better vertex paging

panda/src/display/graphicsEngine.cxx

@@ -44,6 +44,8 @@
 #include "bamCache.h"
 #include "cullableObject.h"
 #include "geomVertexArrayData.h"
+#include "vertexDataSaveFile.h"
+#include "vertexDataBook.h"
 
 #if defined(WIN32)
   #define WINDOWS_LEAN_AND_MEAN
@@ -78,10 +80,12 @@ PStatCollector GraphicsEngine::_delete_pcollector("App:Delete");
 
 
 PStatCollector GraphicsEngine::_sw_sprites_pcollector("SW Sprites");
-PStatCollector GraphicsEngine::_vertex_data_active_pcollector("Vertex Data:Active");
+PStatCollector GraphicsEngine::_vertex_data_small_pcollector("Vertex Data:Small");
+PStatCollector GraphicsEngine::_vertex_data_independent_pcollector("Vertex Data:Independent");
 PStatCollector GraphicsEngine::_vertex_data_resident_pcollector("Vertex Data:Resident");
 PStatCollector GraphicsEngine::_vertex_data_compressed_pcollector("Vertex Data:Compressed");
-PStatCollector GraphicsEngine::_vertex_data_disk_pcollector("Vertex Data:Disk");
+PStatCollector GraphicsEngine::_vertex_data_unused_disk_pcollector("Vertex Data:Disk:Unused");
+PStatCollector GraphicsEngine::_vertex_data_used_disk_pcollector("Vertex Data:Disk:Used");
 
 // These are counted independently by the collision system; we
 // redefine them here so we can reset them at each frame.
@@ -743,13 +747,21 @@ render_frame() {
   _test_geom_pcollector.clear_level();
 
   if (PStatClient::is_connected()) {
-    size_t resident = GeomVertexArrayData::get_global_lru(GeomVertexArrayData::RC_resident)->get_total_size();
-    size_t active = GeomVertexArrayData::get_global_lru(GeomVertexArrayData::RC_resident)->count_active_size();
-    _vertex_data_active_pcollector.set_level(active);
-    _vertex_data_resident_pcollector.set_level(resident - active);
-    _vertex_data_compressed_pcollector.set_level(GeomVertexArrayData::get_global_lru(GeomVertexArrayData::RC_compressed)->get_total_size());
-    _vertex_data_disk_pcollector.set_level(GeomVertexArrayData::get_global_lru(GeomVertexArrayData::RC_disk)->get_total_size());
-    
+    size_t small = GeomVertexArrayData::get_small_lru()->get_total_size();
+    size_t independent = GeomVertexArrayData::get_independent_lru()->get_total_size();
+    size_t resident = VertexDataPage::get_global_lru(VertexDataPage::RC_resident)->get_total_size();
+    size_t compressed = VertexDataPage::get_global_lru(VertexDataPage::RC_compressed)->get_total_size();
+
+    VertexDataSaveFile *save_file = VertexDataPage::get_save_file();
+    size_t total_disk = save_file->get_total_file_size();
+    size_t used_disk = save_file->get_used_file_size();
+
+    _vertex_data_small_pcollector.set_level(small);
+    _vertex_data_independent_pcollector.set_level(independent);
+    _vertex_data_resident_pcollector.set_level(resident);
+    _vertex_data_compressed_pcollector.set_level(compressed);
+    _vertex_data_unused_disk_pcollector.set_level(total_disk - used_disk);
+    _vertex_data_used_disk_pcollector.set_level(used_disk);
   }
 
 #endif  // DO_PSTATS

panda/src/display/graphicsEngine.h

@@ -360,10 +360,12 @@ private:
   static PStatCollector _delete_pcollector;
 
   static PStatCollector _sw_sprites_pcollector;
-  static PStatCollector _vertex_data_active_pcollector;
+  static PStatCollector _vertex_data_small_pcollector;
+  static PStatCollector _vertex_data_independent_pcollector;
   static PStatCollector _vertex_data_resident_pcollector;
   static PStatCollector _vertex_data_compressed_pcollector;
-  static PStatCollector _vertex_data_disk_pcollector;
+  static PStatCollector _vertex_data_used_disk_pcollector;
+  static PStatCollector _vertex_data_unused_disk_pcollector;
 
   static PStatCollector _cnode_volume_pcollector;
   static PStatCollector _gnode_volume_pcollector;

panda/src/dxgsg8/dxGraphicsStateGuardian8.cxx

@@ -930,9 +930,9 @@ draw_triangles(const GeomPrimitivePipelineReader *reader) {
         (D3DPT_TRIANGLELIST,
          min_vertex, max_vertex,
          reader->get_num_primitives(),
-         reader->get_pointer(),
+         reader->get_read_pointer(),
          index_type,
-         _data_reader->get_array_reader(0)->get_pointer(),
+         _data_reader->get_array_reader(0)->get_read_pointer(),
          _data_reader->get_format()->get_array(0)->get_stride());
     }
   } else {
@@ -949,7 +949,7 @@ draw_triangles(const GeomPrimitivePipelineReader *reader) {
       draw_primitive_up(D3DPT_TRIANGLELIST, reader->get_num_primitives(),
                         reader->get_first_vertex(),
                         reader->get_num_vertices(),
-                        _data_reader->get_array_reader(0)->get_pointer(),
+                        _data_reader->get_array_reader(0)->get_read_pointer(),
                         _data_reader->get_format()->get_array(0)->get_stride());
     }
   }
@@ -990,8 +990,8 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
           (D3DPT_TRIANGLESTRIP,
            min_vertex, max_vertex,
            reader->get_num_vertices() - 2,
-           reader->get_pointer(), index_type,
-           _data_reader->get_array_reader(0)->get_pointer(),
+           reader->get_read_pointer(), index_type,
+           _data_reader->get_array_reader(0)->get_read_pointer(),
            _data_reader->get_format()->get_array(0)->get_stride());
       }
     } else {
@@ -1008,7 +1008,7 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
                           reader->get_num_vertices() - 2,
                           reader->get_first_vertex(),
                           reader->get_num_vertices(),
-                          _data_reader->get_array_reader(0)->get_pointer(),
+                          _data_reader->get_array_reader(0)->get_read_pointer(),
                           _data_reader->get_format()->get_array(0)->get_stride());
       }
     }
@@ -1050,9 +1050,9 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
 
       } else {
         // Indexed, client arrays, individual triangle strips.
-        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
         int stride = _data_reader->get_format()->get_array(0)->get_stride();
-        const unsigned char *vertices = reader->get_pointer();
+        const unsigned char *vertices = reader->get_read_pointer();
         D3DFORMAT index_type = get_index_type(reader->get_index_type());
 
         unsigned int start = 0;
@@ -1087,7 +1087,7 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
 
       } else {
         // Nonindexed, client arrays, individual triangle strips.
-        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
         int stride = _data_reader->get_format()->get_array(0)->get_stride();
 
         unsigned int start = 0;
@@ -1150,9 +1150,9 @@ draw_trifans(const GeomPrimitivePipelineReader *reader) {
 
     } else {
       // Indexed, client arrays.
-      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
       int stride = _data_reader->get_format()->get_array(0)->get_stride();
-      const unsigned char *vertices = reader->get_pointer();
+      const unsigned char *vertices = reader->get_read_pointer();
       D3DFORMAT index_type = get_index_type(reader->get_index_type());
 
       unsigned int start = 0;
@@ -1187,7 +1187,7 @@ draw_trifans(const GeomPrimitivePipelineReader *reader) {
 
     } else {
       // Nonindexed, client arrays.
-      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
       int stride = _data_reader->get_format()->get_array(0)->get_stride();
 
       unsigned int start = 0;
@@ -1238,9 +1238,9 @@ draw_lines(const GeomPrimitivePipelineReader *reader) {
         (D3DPT_LINELIST,
          min_vertex, max_vertex,
          reader->get_num_primitives(),
-         reader->get_pointer(),
+         reader->get_read_pointer(),
          index_type,
-         _data_reader->get_array_reader(0)->get_pointer(),
+         _data_reader->get_array_reader(0)->get_read_pointer(),
          _data_reader->get_format()->get_array(0)->get_stride());
     }
   } else {
@@ -1256,7 +1256,7 @@ draw_lines(const GeomPrimitivePipelineReader *reader) {
       draw_primitive_up(D3DPT_LINELIST, reader->get_num_primitives(),
                         reader->get_first_vertex(),
                         reader->get_num_vertices(),
-                        _data_reader->get_array_reader(0)->get_pointer(),
+                        _data_reader->get_array_reader(0)->get_read_pointer(),
                         _data_reader->get_format()->get_array(0)->get_stride());
     }
   }
@@ -1299,7 +1299,7 @@ draw_points(const GeomPrimitivePipelineReader *reader) {
     draw_primitive_up(D3DPT_POINTLIST, reader->get_num_primitives(),
                       reader->get_first_vertex(),
                       reader->get_num_vertices(),
-                      _data_reader->get_array_reader(0)->get_pointer(),
+                      _data_reader->get_array_reader(0)->get_read_pointer(),
                       _data_reader->get_format()->get_array(0)->get_stride());
   }
 }

panda/src/dxgsg8/dxIndexBufferContext8.cxx

@@ -131,7 +131,7 @@ upload_data(const GeomPrimitivePipelineReader *reader) {
   }
 
   GraphicsStateGuardian::_data_transferred_pcollector.add_level(data_size);
-  memcpy(local_pointer, reader->get_pointer(), data_size);
+  memcpy(local_pointer, reader->get_read_pointer(), data_size);
 
   _ibuffer->Unlock();
 }

panda/src/dxgsg8/dxVertexBufferContext8.cxx

@@ -257,7 +257,7 @@ upload_data(const GeomVertexArrayDataHandle *reader) {
   }
 
   GraphicsStateGuardian::_data_transferred_pcollector.add_level(data_size);
-  memcpy(local_pointer, reader->get_pointer(), data_size);
+  memcpy(local_pointer, reader->get_read_pointer(), data_size);
 
   _vbuffer->Unlock();
 }

panda/src/dxgsg9/dxGraphicsStateGuardian9.cxx

@@ -1422,9 +1422,9 @@ draw_triangles(const GeomPrimitivePipelineReader *reader) {
         (D3DPT_TRIANGLELIST,
          min_vertex, max_vertex,
          reader->get_num_primitives(),
-         reader->get_pointer(),
+         reader->get_read_pointer(),
          index_type,
-         _data_reader->get_array_reader(0)->get_pointer(),
+         _data_reader->get_array_reader(0)->get_read_pointer(),
          _data_reader->get_format()->get_array(0)->get_stride());
     }
   } else {
@@ -1446,7 +1446,7 @@ draw_triangles(const GeomPrimitivePipelineReader *reader) {
       draw_primitive_up(D3DPT_TRIANGLELIST, reader->get_num_primitives(),
       reader->get_first_vertex(),
       reader->get_num_vertices(),
-      _data_reader->get_array_reader(0)->get_pointer(),
+      _data_reader->get_array_reader(0)->get_read_pointer(),
       _data_reader->get_format()->get_array(0)->get_stride());
     }
   }
@@ -1497,8 +1497,8 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
           (D3DPT_TRIANGLESTRIP,
            min_vertex, max_vertex,
            reader->get_num_vertices() - 2,
-           reader->get_pointer(), index_type,
-           _data_reader->get_array_reader(0)->get_pointer(),
+           reader->get_read_pointer(), index_type,
+           _data_reader->get_array_reader(0)->get_read_pointer(),
            _data_reader->get_format()->get_array(0)->get_stride());
       }
     } else {
@@ -1518,7 +1518,7 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
         reader->get_num_vertices() - 2,
         reader->get_first_vertex(),
         reader->get_num_vertices(),
-        _data_reader->get_array_reader(0)->get_pointer(),
+        _data_reader->get_array_reader(0)->get_read_pointer(),
         _data_reader->get_format()->get_array(0)->get_stride());
       }
     }
@@ -1561,9 +1561,9 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
 
       } else {
         // Indexed, client arrays, individual triangle strips.
-        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
         int stride = _data_reader->get_format()->get_array(0)->get_stride();
-        const unsigned char *vertices = reader->get_pointer();
+        const unsigned char *vertices = reader->get_read_pointer();
         D3DFORMAT index_type = get_index_type(reader->get_index_type());
 
         unsigned int start = 0;
@@ -1598,7 +1598,7 @@ draw_tristrips(const GeomPrimitivePipelineReader *reader) {
 
       } else {
         // Nonindexed, client arrays, individual triangle strips.
-        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+        const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
         int stride = _data_reader->get_format()->get_array(0)->get_stride();
 
         unsigned int start = 0;
@@ -1664,9 +1664,9 @@ draw_trifans(const GeomPrimitivePipelineReader *reader) {
 
     } else {
       // Indexed, client arrays.
-      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
       int stride = _data_reader->get_format()->get_array(0)->get_stride();
-      const unsigned char *vertices = reader->get_pointer();
+      const unsigned char *vertices = reader->get_read_pointer();
       D3DFORMAT index_type = get_index_type(reader->get_index_type());
 
       unsigned int start = 0;
@@ -1701,7 +1701,7 @@ draw_trifans(const GeomPrimitivePipelineReader *reader) {
 
     } else {
       // Nonindexed, client arrays.
-      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_pointer();
+      const unsigned char *array_data = _data_reader->get_array_reader(0)->get_read_pointer();
       int stride = _data_reader->get_format()->get_array(0)->get_stride();
 
       unsigned int start = 0;
@@ -1753,9 +1753,9 @@ draw_lines(const GeomPrimitivePipelineReader *reader) {
         (D3DPT_LINELIST,
          min_vertex, max_vertex,
          reader->get_num_primitives(),
-         reader->get_pointer(),
+         reader->get_read_pointer(),
          index_type,
-         _data_reader->get_array_reader(0)->get_pointer(),
+         _data_reader->get_array_reader(0)->get_read_pointer(),
          _data_reader->get_format()->get_array(0)->get_stride());
     }
   } else {
@@ -1771,7 +1771,7 @@ draw_lines(const GeomPrimitivePipelineReader *reader) {
       draw_primitive_up(D3DPT_LINELIST, reader->get_num_primitives(),
       reader->get_first_vertex(),
       reader->get_num_vertices(),
-      _data_reader->get_array_reader(0)->get_pointer(),
+      _data_reader->get_array_reader(0)->get_read_pointer(),
       _data_reader->get_format()->get_array(0)->get_stride());
     }
   }
@@ -1814,7 +1814,7 @@ draw_points(const GeomPrimitivePipelineReader *reader) {
     draw_primitive_up(D3DPT_POINTLIST, reader->get_num_primitives(),
                       reader->get_first_vertex(),
                       reader->get_num_vertices(),
-                      _data_reader->get_array_reader(0)->get_pointer(),
+                      _data_reader->get_array_reader(0)->get_read_pointer(),
                       _data_reader->get_format()->get_array(0)->get_stride());
   }
 }

panda/src/dxgsg9/dxIndexBufferContext9.cxx

@@ -237,7 +237,7 @@ upload_data(const GeomPrimitivePipelineReader *reader) {
   }
 
   GraphicsStateGuardian::_data_transferred_pcollector.add_level(data_size);
-  memcpy(local_pointer, reader->get_pointer(), data_size);
+  memcpy(local_pointer, reader->get_read_pointer(), data_size);
 
   _ibuffer->Unlock();
 }

panda/src/dxgsg9/dxVertexBufferContext9.cxx

@@ -473,7 +473,7 @@ upload_data(const GeomVertexArrayDataHandle *reader) {
   }
 
   GraphicsStateGuardian::_data_transferred_pcollector.add_level(data_size);
-  memcpy(local_pointer, reader->get_pointer(), data_size);
+  memcpy(local_pointer, reader->get_read_pointer(), data_size);
 
   _vbuffer->Unlock();
 }

panda/src/glstuff/glGraphicsStateGuardian_src.cxx

@@ -2726,12 +2726,12 @@ apply_vertex_buffer(VertexBufferContext *vbc,
     if (num_bytes != 0) {
       if (gvbc->changed_size(reader) || gvbc->changed_usage_hint(reader)) {
         _glBufferData(GL_ARRAY_BUFFER, num_bytes,
-                      reader->get_pointer(),
+                      reader->get_read_pointer(),
                       get_usage(reader->get_usage_hint()));
 
       } else {
         _glBufferSubData(GL_ARRAY_BUFFER, 0, num_bytes,
-                         reader->get_pointer());
+                         reader->get_read_pointer());
       }
       _data_transferred_pcollector.add_level(num_bytes);
     }
@@ -2800,7 +2800,7 @@ const unsigned char *CLP(GraphicsStateGuardian)::
 setup_array_data(const GeomVertexArrayDataHandle *array_reader) {
   if (!_supports_buffers) {
     // No support for buffer objects; always render from client.
-    return array_reader->get_pointer();
+    return array_reader->get_read_pointer();
   }
   if (!vertex_buffers || _geom_display_list != 0 ||
       array_reader->get_usage_hint() == Geom::UH_client) {
@@ -2814,12 +2814,12 @@ setup_array_data(const GeomVertexArrayDataHandle *array_reader) {
       _glBindBuffer(GL_ARRAY_BUFFER, 0);
       _current_vbuffer_index = 0;
     }
-    return array_reader->get_pointer();
+    return array_reader->get_read_pointer();
   }
 
   // Prepare the buffer object and bind it.
   VertexBufferContext *vbc = ((GeomVertexArrayData *)array_reader->get_object())->prepare_now(get_prepared_objects(), this);
-  nassertr(vbc != (VertexBufferContext *)NULL, array_reader->get_pointer());
+  nassertr(vbc != (VertexBufferContext *)NULL, array_reader->get_read_pointer());
   apply_vertex_buffer(vbc, array_reader);
 
   // NULL is the OpenGL convention for the first byte of the buffer object.
@@ -2896,12 +2896,12 @@ apply_index_buffer(IndexBufferContext *ibc,
     if (num_bytes != 0) {
       if (gibc->changed_size(reader) || gibc->changed_usage_hint(reader)) {
         _glBufferData(GL_ELEMENT_ARRAY_BUFFER, num_bytes,
-                      reader->get_pointer(),
+                      reader->get_read_pointer(),
                       get_usage(reader->get_usage_hint()));
 
       } else {
         _glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, num_bytes,
-                         reader->get_pointer());
+                         reader->get_read_pointer());
       }
       _data_transferred_pcollector.add_level(num_bytes);
     }
@@ -2969,7 +2969,7 @@ const unsigned char *CLP(GraphicsStateGuardian)::
 setup_primitive(const GeomPrimitivePipelineReader *reader) {
   if (!_supports_buffers) {
     // No support for buffer objects; always render from client.
-    return reader->get_pointer();
+    return reader->get_read_pointer();
   }
   if (!vertex_buffers || _geom_display_list != 0 ||
       reader->get_usage_hint() == Geom::UH_client) {
@@ -2983,12 +2983,12 @@ setup_primitive(const GeomPrimitivePipelineReader *reader) {
       _glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0);
       _current_ibuffer_index = 0;
     }
-    return reader->get_pointer();
+    return reader->get_read_pointer();
   }
 
   // Prepare the buffer object and bind it.
   IndexBufferContext *ibc = ((GeomPrimitive *)reader->get_object())->prepare_now(get_prepared_objects(), this);
-  nassertr(ibc != (IndexBufferContext *)NULL, reader->get_pointer());
+  nassertr(ibc != (IndexBufferContext *)NULL, reader->get_read_pointer());
   apply_index_buffer(ibc, reader);
 
   // NULL is the OpenGL convention for the first byte of the buffer object.

panda/src/gobj/Sources.pp

@@ -64,6 +64,8 @@
     userVertexSlider.I userVertexSlider.h \
     userVertexTransform.I userVertexTransform.h \
     vertexBufferContext.I vertexBufferContext.h \
+    vertexDataBook.I vertexDataBook.h \
+    vertexDataBuffer.I vertexDataBuffer.h \
     vertexDataSaveFile.I vertexDataSaveFile.h \
     vertexSlider.I vertexSlider.h \
     vertexTransform.I vertexTransform.h \
@@ -121,6 +123,8 @@
     userVertexSlider.cxx \
     userVertexTransform.cxx \
     vertexBufferContext.cxx \
+    vertexDataBook.cxx \
+    vertexDataBuffer.cxx \
     vertexDataSaveFile.cxx \
     vertexSlider.cxx \
     vertexTransform.cxx \
@@ -182,6 +186,8 @@
     userVertexSlider.I userVertexSlider.h \
     userVertexTransform.I userVertexTransform.h \
     vertexBufferContext.I vertexBufferContext.h \
+    vertexDataBook.I vertexDataBook.h \
+    vertexDataBuffer.I vertexDataBuffer.h \
     vertexDataSaveFile.I vertexDataSaveFile.h \
     vertexSlider.I vertexSlider.h \
     vertexTransform.I vertexTransform.h \

panda/src/gobj/config_gobj.cxx

@@ -51,6 +51,8 @@
 #include "transformTable.h"
 #include "userVertexSlider.h"
 #include "userVertexTransform.h"
+#include "vertexDataBook.h"
+#include "vertexDataBuffer.h"
 #include "vertexTransform.h"
 #include "vertexSlider.h"
 #include "videoTexture.h"
@@ -284,6 +286,12 @@ ConfigVariableString vertex_save_file_prefix
           "been evicted from RAM.  A uniquifying sequence number and "
           "filename extension will be appended to this string."));
 
+ConfigVariableInt vertex_data_small_size
+("vertex-data-small-size", 64,
+ PRC_DESC("When a GeomVertexArrayData is this number of bytes or smaller, it "
+          "is deemed too small to pay the overhead of paging it in and out, "
+          "and it is permanently retained resident."));
+
 
 
 ConfigureFn(config_gobj) {
@@ -330,6 +338,8 @@ ConfigureFn(config_gobj) {
   UserVertexTransform::init_type();
   VertexBufferContext::init_type();
   VertexSlider::init_type();
+  VertexDataBuffer::init_type();
+  VertexDataPage::init_type();
   VertexTransform::init_type();
   VideoTexture::init_type();
 

panda/src/gobj/config_gobj.h

@@ -63,15 +63,16 @@ extern EXPCL_PANDA ConfigVariableBool textures_header_only;
 extern EXPCL_PANDA ConfigVariableInt geom_cache_size;
 extern EXPCL_PANDA ConfigVariableInt geom_cache_min_frames;
 
-extern ConfigVariableDouble default_near;
-extern ConfigVariableDouble default_far;
-extern ConfigVariableDouble default_fov;
-extern ConfigVariableDouble default_iod;
-extern ConfigVariableDouble default_converge;
-extern ConfigVariableDouble default_keystone;
+extern EXPCL_PANDA ConfigVariableDouble default_near;
+extern EXPCL_PANDA ConfigVariableDouble default_far;
+extern EXPCL_PANDA ConfigVariableDouble default_fov;
+extern EXPCL_PANDA ConfigVariableDouble default_iod;
+extern EXPCL_PANDA ConfigVariableDouble default_converge;
+extern EXPCL_PANDA ConfigVariableDouble default_keystone;
 
-extern ConfigVariableFilename vertex_save_file_directory;
-extern ConfigVariableString vertex_save_file_prefix;
+extern EXPCL_PANDA ConfigVariableFilename vertex_save_file_directory;
+extern EXPCL_PANDA ConfigVariableString vertex_save_file_prefix;
+extern EXPCL_PANDA ConfigVariableInt vertex_data_small_size;
 
 #endif
 

panda/src/gobj/geomEnums.h

@@ -209,19 +209,6 @@ PUBLISHED:
     AT_panda,    // Vertex animation calculated on the CPU by Panda.
     AT_hardware, // Hardware-accelerated animation on the graphics card.
   };
-
-  // These are used to indicate the current residency state of vertex
-  // data, which may or may not have been temporarily evicted to
-  // satisfy memory requirements.
-  enum RamClass {
-    RC_resident,
-    RC_compressed,
-    RC_disk,
-    RC_compressed_disk,
-
-    RC_end_of_list,  // list marker; do not use
-  };
-
 };
 
 EXPCL_PANDA ostream &operator << (ostream &out, GeomEnums::NumericType numeric_type);

panda/src/gobj/geomPrimitive.I

@@ -661,13 +661,13 @@ get_vertices_reader() const {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomPrimitivePipelineReader::get_pointer
+//     Function: GeomPrimitivePipelineReader::get_read_pointer
 //       Access: Public
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 INLINE const unsigned char *GeomPrimitivePipelineReader::
-get_pointer() const {
-  return _vertices_reader->get_pointer();
+get_read_pointer() const {
+  return _vertices_reader->get_read_pointer();
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/gobj/geomPrimitive.h

@@ -346,7 +346,7 @@ public:
   bool check_valid(const GeomVertexDataPipelineReader *data_reader) const;
   INLINE int get_index_stride() const;
   INLINE const GeomVertexArrayDataHandle *get_vertices_reader() const;
-  INLINE const unsigned char *get_pointer() const;
+  INLINE const unsigned char *get_read_pointer() const;
   INLINE CPTA_int get_ends() const;
   INLINE CPT(GeomVertexArrayData) get_mins() const;
   INLINE CPT(GeomVertexArrayData) get_maxs() const;

panda/src/gobj/geomVertexArrayData.I

@@ -129,7 +129,7 @@ clear_rows() {
 INLINE int GeomVertexArrayData::
 get_data_size_bytes() const {
   CDReader cdata(_cycler);
-  return cdata->_data_full_size;
+  return cdata->_buffer.get_size();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -178,54 +178,54 @@ modify_handle(Thread *current_thread) {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::get_ram_class
-//       Access: Published
-//  Description: Returns the current ram class of the array.  If this
-//               is other than RC_resident, the array data is not
-//               resident in memory.
+//     Function: GeomVertexArrayData::get_independent_lru
+//       Access: Published, Static
+//  Description: Returns a pointer to the global LRU object that
+//               manages the GeomVertexArrayData's that have not (yet)
+//               been paged out.
 ////////////////////////////////////////////////////////////////////
-INLINE GeomVertexArrayData::RamClass GeomVertexArrayData::
-get_ram_class() const {
-  return _ram_class;
+INLINE SimpleLru *GeomVertexArrayData::
+get_independent_lru() {
+  return &_independent_lru;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::get_global_lru
+//     Function: GeomVertexArrayData::get_small_lru
 //       Access: Published, Static
 //  Description: Returns a pointer to the global LRU object that
-//               manages the GeomVertexArrayData's with the indicated
-//               RamClass.
+//               manages the GeomVertexArrayData's that are deemed too
+//               small to be paged out.
 ////////////////////////////////////////////////////////////////////
 INLINE SimpleLru *GeomVertexArrayData::
-get_global_lru(RamClass rclass) {
-  nassertr(rclass >= 0 && rclass < RC_end_of_list, NULL);
-  return _global_lru[rclass];
+get_small_lru() {
+  return &_small_lru;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::get_save_file
+//     Function: GeomVertexArrayData::get_book
 //       Access: Published, Static
-//  Description: Returns the global VertexDataSaveFile that will be
-//               used to save vertex data buffers to disk when
-//               necessary.
-////////////////////////////////////////////////////////////////////
-INLINE VertexDataSaveFile *GeomVertexArrayData::
-get_save_file() {
-  if (_save_file == (VertexDataSaveFile *)NULL) {
-    make_save_file();
-  }
-  return _save_file;
+//  Description: Returns the global VertexDataBook that will be
+//               used to allocate vertex data buffers.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBook &GeomVertexArrayData::
+get_book() {
+  return _book;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::set_ram_class
+//     Function: GeomVertexArrayData::set_lru_size
 //       Access: Private
-//  Description: Puts the data in a new ram class.
+//  Description: Should be called when the size of the buffer changes.
 ////////////////////////////////////////////////////////////////////
 INLINE void GeomVertexArrayData::
-set_ram_class(RamClass rclass) {
-  _ram_class = rclass;
-  mark_used_lru(_global_lru[rclass]);
+set_lru_size(size_t lru_size) {
+  SimpleLruPage::set_lru_size(lru_size);
+
+  if (lru_size <= vertex_data_small_size) {
+    SimpleLruPage::mark_used_lru(&_small_lru);
+  } else {
+    SimpleLruPage::mark_used_lru(&_independent_lru);
+  }
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -235,9 +235,7 @@ set_ram_class(RamClass rclass) {
 ////////////////////////////////////////////////////////////////////
 INLINE GeomVertexArrayData::CData::
 CData() :
-  _usage_hint(UH_unspecified),
-  _data(GeomVertexArrayData::get_class_type()),
-  _data_full_size(0)
+  _usage_hint(UH_unspecified)
 {
 }
 
@@ -249,8 +247,7 @@ CData() :
 INLINE GeomVertexArrayData::CData::
 CData(const GeomVertexArrayData::CData &copy) :
   _usage_hint(copy._usage_hint),
-  _data(copy._data),
-  _data_full_size(copy._data_full_size),
+  _buffer(copy._buffer),
   _modified(copy._modified)
 {
 }
@@ -263,9 +260,8 @@ CData(const GeomVertexArrayData::CData &copy) :
 INLINE void GeomVertexArrayData::CData::
 operator = (const GeomVertexArrayData::CData &copy) {
   _usage_hint = copy._usage_hint;
+  _buffer = copy._buffer;
   _modified = copy._modified;
-  _data = copy._data;
-  _data_full_size = copy._data_full_size;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -370,34 +366,28 @@ get_object() {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayDataHandle::get_pointer
+//     Function: GeomVertexArrayDataHandle::get_read_pointer
 //       Access: Public
-//  Description: Returns a pointer to the beginning of the actual data
-//               stream.
+//  Description: Returns a readable pointer to the beginning of the
+//               actual data stream.
 ////////////////////////////////////////////////////////////////////
 INLINE const unsigned char *GeomVertexArrayDataHandle::
-get_pointer() const {
+get_read_pointer() const {
   check_resident();
-  if (_cdata->_data.size() == 0) {
-    return NULL;
-  }
-  return &_cdata->_data[0];
+  return _cdata->_buffer.get_read_pointer();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayDataHandle::get_pointer
+//     Function: GeomVertexArrayDataHandle::get_write_pointer
 //       Access: Public
-//  Description: Returns a pointer to the beginning of the actual data
-//               stream.
+//  Description: Returns a writable pointer to the beginning of the
+//               actual data stream.
 ////////////////////////////////////////////////////////////////////
 INLINE unsigned char *GeomVertexArrayDataHandle::
-get_pointer() {
+get_write_pointer() {
   nassertr(_writable, NULL);
   check_resident();
-  if (_cdata->_data.size() == 0) {
-    return NULL;
-  }
-  return &_cdata->_data[0];
+  return _cdata->_buffer.get_write_pointer();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -447,7 +437,7 @@ clear_rows() {
 ////////////////////////////////////////////////////////////////////
 INLINE int GeomVertexArrayDataHandle::
 get_data_size_bytes() const {
-  return _cdata->_data_full_size;
+  return _cdata->_buffer.get_size();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -460,6 +450,36 @@ get_modified() const {
   return _cdata->_modified;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: GeomVertexArrayDataHandle::get_data
+//       Access: Published
+//  Description: Returns the entire raw data of the
+//               GeomVertexArrayData object, formatted as a string.
+//               This is primarily for the benefit of high-level
+//               languages such as Python.
+////////////////////////////////////////////////////////////////////
+INLINE string GeomVertexArrayDataHandle::
+get_data() const {
+  check_resident();
+  return string((const char *)_cdata->_buffer.get_read_pointer(), _cdata->_buffer.get_size());
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: GeomVertexArrayDataHandle::get_subdata
+//       Access: Published
+//  Description: Returns a subset of the raw data of the
+//               GeomVertexArrayData object, formatted as a string.
+//               This is primarily for the benefit of high-level
+//               languages such as Python.
+////////////////////////////////////////////////////////////////////
+INLINE string GeomVertexArrayDataHandle::
+get_subdata(size_t start, size_t size) const {
+  check_resident();
+  start = min(start, _cdata->_buffer.get_size());
+  size = min(size, _cdata->_buffer.get_size() - start);
+  return string((const char *)_cdata->_buffer.get_read_pointer() + start, size);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: GeomVertexArrayDataHandle::check_resident
 //       Access: Published
@@ -468,12 +488,7 @@ get_modified() const {
 ////////////////////////////////////////////////////////////////////
 void GeomVertexArrayDataHandle::
 check_resident() const {
-  if (_object->get_ram_class() != RC_resident) {
-    _object->make_resident();
-  } else {
-    _object->mark_used_lru();
-  }
-  nassertv(_cdata->_data.size() == _cdata->_data_full_size);
+  _object->mark_used_lru();
 }
 
 INLINE ostream &

panda/src/gobj/geomVertexArrayData.cxx

@@ -26,69 +26,27 @@
 #include "config_gobj.h"
 #include "pStatTimer.h"
 #include "configVariableInt.h"
-#include "vertexDataSaveFile.h"
 #include "simpleAllocator.h"
+#include "vertexDataBuffer.h"
 
-#ifdef HAVE_ZLIB
-#include <zlib.h>
-#endif
-
-ConfigVariableInt max_ram_vertex_data
-("max-ram-vertex-data", -1,
+ConfigVariableInt max_independent_vertex_data
+("max-independent-vertex-data", -1,
  PRC_DESC("Specifies the maximum number of bytes of all vertex data "
-          "that is allowed to remain resident in system RAM at one time. "
-          "If more than this number of bytes of vertices are created, "
-          "the least-recently-used ones will be temporarily compressed in "
-          "system RAM until they are needed.  Set it to -1 for no limit."));
+          "that is independent of the paging system.  This is an "
+          "initial buffer before max-ram-vertex-data, specifically "
+          "designed for vertex datas that are dynamic in nature and "
+          "may change size or be created and destroyed frequently."));
 
-ConfigVariableInt max_compressed_vertex_data
-("max-compressed-vertex-data", -1,
- PRC_DESC("Specifies the maximum number of bytes of all vertex data "
-          "that is allowed to remain compressed in system RAM at one time. "
-          "If more than this number of bytes of vertices are created, "
-          "the least-recently-used ones will be temporarily flushed to "
-          "disk until they are needed.  Set it to -1 for no limit."));
-
-ConfigVariableInt vertex_data_compression_level
-("vertex-data-compression-level", 1,
- PRC_DESC("Specifies the zlib compression level to use when compressing "
-          "vertex data.  The number should be in the range 1 to 9, where "
-          "larger values are slower but give better compression."));
-
-ConfigVariableInt max_disk_vertex_data
-("max-disk-vertex-data", -1,
- PRC_DESC("Specifies the maximum number of bytes of vertex data "
-          "that is allowed to be written to disk.  Set it to -1 for no "
-          "limit."));
-
-// We make this a static constant rather than a dynamic variable,
-// since we can't tolerate this value changing at runtime.
-static const size_t min_vertex_data_compress_size = 
-  ConfigVariableInt
-  ("min-vertex-data-compress-size", 64,
-   PRC_DESC("This is the minimum number of bytes that we deem worthy of "
-            "passing through zlib to compress, when a vertex buffer is "
-            "evicted from resident state and compressed for long-term "
-            "storage.  Buffers smaller than this are assumed to be likely to "
-            "have minimal compression gains (or even end up larger)."));
-
-SimpleLru GeomVertexArrayData::_ram_lru(max_ram_vertex_data);
-SimpleLru GeomVertexArrayData::_compressed_lru(max_compressed_vertex_data);
-SimpleLru GeomVertexArrayData::_disk_lru(0);
-
-SimpleLru *GeomVertexArrayData::_global_lru[RC_end_of_list] = {
-  &GeomVertexArrayData::_ram_lru,
-  &GeomVertexArrayData::_compressed_lru,
-  &GeomVertexArrayData::_disk_lru,
-  &GeomVertexArrayData::_disk_lru,
-};
-
-VertexDataSaveFile *GeomVertexArrayData::_save_file;
-
-PStatCollector GeomVertexArrayData::_vdata_compress_pcollector("*:Vertex Data:Compress");
-PStatCollector GeomVertexArrayData::_vdata_decompress_pcollector("*:Vertex Data:Decompress");
-PStatCollector GeomVertexArrayData::_vdata_save_pcollector("*:Vertex Data:Save");
-PStatCollector GeomVertexArrayData::_vdata_restore_pcollector("*:Vertex Data:Restore");
+ConfigVariableInt vertex_data_page_size
+("vertex-data-page-size", 512,
+ PRC_DESC("The number of bytes to allocate at a time for vertex data.  "
+          "This also controls the page size that is compressed and written "
+          "to disk when vertex data pages are evicted from memory."));
+
+SimpleLru GeomVertexArrayData::_independent_lru(max_independent_vertex_data);
+SimpleLru GeomVertexArrayData::_small_lru(max_independent_vertex_data);
+
+VertexDataBook GeomVertexArrayData::_book(vertex_data_page_size);
 
 
 TypeHandle GeomVertexArrayData::_type_handle;
@@ -104,8 +62,6 @@ TypeHandle GeomVertexArrayDataHandle::_type_handle;
 GeomVertexArrayData::
 GeomVertexArrayData() : SimpleLruPage(0) {
   _endian_reversed = false;
-  _ram_class = RC_resident;
-  _saved_block = NULL;
 
   // Can't put it in the LRU until it has been read in and made valid.
 }
@@ -117,7 +73,6 @@ GeomVertexArrayData() : SimpleLruPage(0) {
 ////////////////////////////////////////////////////////////////////
 PT(CopyOnWriteObject) GeomVertexArrayData::
 make_cow_copy() {
-  make_resident();
   return new GeomVertexArrayData(*this);
 }
 
@@ -140,10 +95,7 @@ GeomVertexArrayData(const GeomVertexArrayFormat *array_format,
 
   _endian_reversed = false;
 
-  _ram_class = RC_resident;
-  _saved_block = NULL;
-  mark_used_lru(_global_lru[RC_resident]);
-
+  set_lru_size(0);
   nassertv(_array_format->is_registered());
 }
   
@@ -161,10 +113,9 @@ GeomVertexArrayData(const GeomVertexArrayData &copy) :
 {
   _endian_reversed = false;
 
-  _ram_class = copy._ram_class;
-  _saved_block = NULL;
-  mark_used_lru(_global_lru[_ram_class]);
+  copy.mark_used_lru();
 
+  set_lru_size(get_data_size_bytes());
   nassertv(_array_format->is_registered());
 }
 
@@ -180,6 +131,9 @@ void GeomVertexArrayData::
 operator = (const GeomVertexArrayData &copy) {
   CopyOnWriteObject::operator = (copy);
   SimpleLruPage::operator = (copy);
+
+  copy.mark_used_lru();
+
   _array_format = copy._array_format;
   _cycler = copy._cycler;
 
@@ -200,10 +154,6 @@ operator = (const GeomVertexArrayData &copy) {
 GeomVertexArrayData::
 ~GeomVertexArrayData() {
   release_all();
-
-  if (_saved_block != (SimpleAllocatorBlock *)NULL) {
-    delete _saved_block;
-  }
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -369,208 +319,9 @@ release_all() {
 ////////////////////////////////////////////////////////////////////
 void GeomVertexArrayData::
 lru_epoch() {
-  _ram_lru.begin_epoch();
-  _compressed_lru.begin_epoch();
-
-  // No automatic eviction from the Disk LRU.
-  //_disk_lru.begin_epoch();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::make_resident
-//       Access: Published
-//  Description: Moves the vertex data to fully resident status by
-//               expanding it or reading it from disk as necessary.
-////////////////////////////////////////////////////////////////////
-void GeomVertexArrayData::
-make_resident() {
-  // TODO: make this work with pipelining properly.
-
-  if (_ram_class == RC_resident) {
-    // If we're already resident, just mark the page recently used.
-    mark_used_lru();
-    return;
-  }
-
-  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
-    restore_from_disk();
-  }
-
-  if (_ram_class == RC_compressed) {
-    CDWriter cdata(_cycler, true);
-#ifdef HAVE_ZLIB
-    if (cdata->_data_full_size > min_vertex_data_compress_size) {
-      PStatTimer timer(_vdata_decompress_pcollector);
-
-      if (gobj_cat.is_debug()) {
-        gobj_cat.debug()
-          << "Expanding " << *this << " from " << cdata->_data.size()
-          << " to " << cdata->_data_full_size << "\n";
-      }
-      Data new_data(cdata->_data_full_size, get_class_type());
-      uLongf dest_len = cdata->_data_full_size;
-      int result = uncompress(&new_data[0], &dest_len,
-                              &cdata->_data[0], cdata->_data.size());
-      if (result != Z_OK) {
-        gobj_cat.error()
-          << "Couldn't expand: zlib error " << result << "\n";
-        nassert_raise("zlib error");
-      }
-      nassertv(dest_len == new_data.size());
-      cdata->_data.swap(new_data);
-    }
-#endif
-    set_lru_size(cdata->_data.size());
-    set_ram_class(RC_resident);
-  }
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::make_compressed
-//       Access: Published
-//  Description: Moves the vertex data to compressed status by
-//               compressing it or reading it from disk as necessary.
-////////////////////////////////////////////////////////////////////
-void GeomVertexArrayData::
-make_compressed() {
-  // TODO: make this work with pipelining properly.
-
-  if (_ram_class == RC_compressed) {
-    // If we're already compressed, just mark the page recently used.
-    mark_used_lru();
-    return;
-  }
-
-  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
-    restore_from_disk();
-  }
-
-  if (_ram_class == RC_resident) {
-    CDWriter cdata(_cycler, true);
-#ifdef HAVE_ZLIB
-    if (cdata->_data_full_size > min_vertex_data_compress_size) {
-      PStatTimer timer(_vdata_compress_pcollector);
-
-      // According to the zlib manual, we need to provide this much
-      // buffer to the compress algorithm: 0.1% bigger plus twelve
-      // bytes.
-      uLongf buffer_size = cdata->_data_full_size + ((cdata->_data_full_size + 999) / 1000) + 12;
-      Bytef *buffer = new Bytef[buffer_size];
-
-      int result = compress2(buffer, &buffer_size,
-                             &cdata->_data[0], cdata->_data_full_size,
-                             vertex_data_compression_level);
-      if (result != Z_OK) {
-        gobj_cat.error()
-          << "Couldn't compress: zlib error " << result << "\n";
-        nassert_raise("zlib error");
-      }
-    
-      Data new_data(buffer_size, get_class_type());
-      memcpy(&new_data[0], buffer, buffer_size);
-      delete[] buffer;
-
-      cdata->_data.swap(new_data);
-      if (gobj_cat.is_debug()) {
-        gobj_cat.debug()
-          << "Compressed " << *this << " from " << cdata->_data_full_size
-          << " to " << cdata->_data.size() << "\n";
-      }
-    }
-#endif
-    set_lru_size(cdata->_data.size());
-    set_ram_class(RC_compressed);
-  }
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::make_disk
-//       Access: Published
-//  Description: Moves the vertex data to disk status by
-//               writing it to disk as necessary.
-////////////////////////////////////////////////////////////////////
-void GeomVertexArrayData::
-make_disk() {
-  // TODO: make this work with pipelining properly.
-
-  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
-    // If we're already compressed, just mark the page recently used.
-    mark_used_lru();
-    return;
-  }
-
-  if (_ram_class == RC_resident || _ram_class == RC_compressed) {
-    nassertv(_saved_block == (SimpleAllocatorBlock *)NULL);
-    CDWriter cdata(_cycler, true);
-
-    PStatTimer timer(_vdata_save_pcollector);
-
-    if (gobj_cat.is_debug()) {
-      gobj_cat.debug()
-        << "Storing " << *this << ", " << cdata->_data.size()
-        << " bytes, to disk\n";
-    }
-
-    const unsigned char *data = &cdata->_data[0];
-    size_t size = cdata->_data.size();
-    _saved_block = get_save_file()->write_data(data, size);
-    if (_saved_block == NULL) {
-      // Can't write it to disk.  Too bad.
-      mark_used_lru();
-      return;
-    }
-
-    // We swap the pvector with an empty one, to really make it free
-    // its memory.  Otherwise it might optimistically keep the memory
-    // allocated.
-    Data new_data(get_class_type());
-    cdata->_data.swap(new_data);
-
-    if (_ram_class == RC_resident) {
-      set_ram_class(RC_disk);
-    } else {
-      set_ram_class(RC_compressed_disk);
-    }
-  }
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::restore_from_disk
-//       Access: Published
-//  Description: Restores the vertex data from disk and makes it
-//               either compressed or resident (according to whether
-//               it was stored compressed on disk).
-////////////////////////////////////////////////////////////////////
-void GeomVertexArrayData::
-restore_from_disk() {
-  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
-    nassertv(_saved_block != (SimpleAllocatorBlock *)NULL);
-    CDWriter cdata(_cycler, true);
-
-    PStatTimer timer(_vdata_restore_pcollector);
-
-    size_t size = _saved_block->get_size();
-    if (gobj_cat.is_debug()) {
-      gobj_cat.debug()
-        << "Restoring " << *this << ", " << size
-        << " bytes, from disk\n";
-    }
-
-    Data new_data(size, get_class_type());
-    if (!get_save_file()->read_data(&new_data[0], size, _saved_block)) {
-      nassert_raise("read error");
-    }
-    cdata->_data.swap(new_data);
-
-    delete _saved_block;
-    _saved_block = NULL;
-
-    if (_ram_class == RC_compressed_disk) {
-      set_ram_class(RC_compressed);
-    } else {
-      set_ram_class(RC_resident);
-    }
-  }
+  _independent_lru.begin_epoch();
+  VertexDataPage::get_global_lru(VertexDataPage::RC_resident)->begin_epoch();
+  VertexDataPage::get_global_lru(VertexDataPage::RC_compressed)->begin_epoch();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -590,30 +341,9 @@ restore_from_disk() {
 ////////////////////////////////////////////////////////////////////
 void GeomVertexArrayData::
 evict_lru() {
-  nassertv(get_lru() == get_global_lru(_ram_class));
-
-  switch (_ram_class) {
-  case RC_resident:
-    if (_compressed_lru.get_max_size() == 0) {
-      make_disk();
-    } else {
-      make_compressed();
-    }
-    break;
-
-  case RC_compressed:
-    make_disk();
-    break;
-
-  case RC_disk:
-  case RC_compressed_disk:
-    gobj_cat.warning()
-      << "Cannot evict array data from disk.\n";
-    break;
-
-  case RC_end_of_list:
-    break;
-  }
+  dequeue_lru();
+  CDWriter cdata(_cycler, true);
+  cdata->_buffer.page_out(_book);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -675,21 +405,6 @@ reverse_data_endianness(unsigned char *dest, const unsigned char *source,
   }
 }
 
-////////////////////////////////////////////////////////////////////
-//     Function: GeomVertexArrayData::make_save_file
-//       Access: Private, Static
-//  Description: Creates the global VertexDataSaveFile that will be
-//               used to save vertex data buffers to disk when
-//               necessary.
-////////////////////////////////////////////////////////////////////
-void GeomVertexArrayData::
-make_save_file() {
-  size_t max_size = (size_t)max_disk_vertex_data;
-
-  _save_file = new VertexDataSaveFile(vertex_save_file_directory,
-                                      vertex_save_file_prefix, max_size);
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: GeomVertexArrayData::register_with_read_factory
 //       Access: Public, Static
@@ -709,7 +424,6 @@ register_with_read_factory() {
 ////////////////////////////////////////////////////////////////////
 void GeomVertexArrayData::
 write_datagram(BamWriter *manager, Datagram &dg) {
-  make_resident();
   CopyOnWriteObject::write_datagram(manager, dg);
 
   manager->write_pointer(dg, _array_format);
@@ -776,12 +490,12 @@ finalize(BamReader *manager) {
 
   if (_endian_reversed) {
     // Now is the time to endian-reverse the data.
-    Data new_data(cdata->_data.size(), get_class_type());
-    reverse_data_endianness(&new_data[0], &cdata->_data[0], cdata->_data.size());
-    cdata->_data.swap(new_data);
+    VertexDataBuffer new_buffer(cdata->_buffer.get_size());
+    reverse_data_endianness(new_buffer.get_write_pointer(), cdata->_buffer.get_read_pointer(), cdata->_buffer.get_size());
+    cdata->_buffer.swap(new_buffer);
   }
 
-  set_ram_class(RC_resident);
+  set_lru_size(cdata->_buffer.get_size());
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -850,17 +564,17 @@ write_datagram(BamWriter *manager, Datagram &dg, void *extra_data) const {
   GeomVertexArrayData *array_data = (GeomVertexArrayData *)extra_data;
   dg.add_uint8(_usage_hint);
 
-  dg.add_uint32(_data.size());
+  dg.add_uint32(_buffer.get_size());
 
   if (manager->get_file_endian() == BE_native) {
     // For native endianness, we only have to write the data directly.
-    dg.append_data(&_data[0], _data.size());
+    dg.append_data(_buffer.get_read_pointer(), _buffer.get_size());
 
   } else {
     // Otherwise, we have to convert it.
-    Data new_data(_data.size(), GeomVertexArrayData::get_class_type());
-    array_data->reverse_data_endianness(&new_data[0], &_data[0], _data.size());
-    dg.append_data(&new_data[0], new_data.size());
+    unsigned char *new_data = (unsigned char *)alloca(_buffer.get_size());
+    array_data->reverse_data_endianness(new_data, _buffer.get_read_pointer(), _buffer.get_size());
+    dg.append_data(new_data, _buffer.get_size());
   }
 }
 
@@ -880,18 +594,17 @@ fillin(DatagramIterator &scan, BamReader *manager, void *extra_data) {
     // Before bam version 6.8, the array data was a PTA_uchar.
     PTA_uchar new_data;
     READ_PTA(manager, scan, array_data->read_raw_data, new_data);
-    _data = new_data.v();
+    _buffer.unclean_realloc(new_data.size());
+    memcpy(_buffer.get_write_pointer(), &new_data[0], new_data.size());
 
   } else {
     // Now, the array data is just stored directly.
     size_t size = scan.get_uint32();
-    Data new_data(size, GeomVertexArrayData::get_class_type());
+    _buffer.unclean_realloc(size);
     const unsigned char *source_data = 
       (const unsigned char *)scan.get_datagram().get_data();
-    memcpy(&new_data[0], source_data + scan.get_current_index(), size);
+    memcpy(_buffer.get_write_pointer(), source_data + scan.get_current_index(), size);
     scan.skip_bytes(size);
-
-    _data.swap(new_data);
   }
 
   if (manager->get_file_endian() != BE_native) {
@@ -904,14 +617,13 @@ fillin(DatagramIterator &scan, BamReader *manager, void *extra_data) {
       // Since we have the _array_format pointer now, we can reverse
       // it immediately (and we should, to support threaded CData
       // updates).
-      Data new_data(_data.size(), GeomVertexArrayData::get_class_type());
-      array_data->reverse_data_endianness(&new_data[0], &_data[0], _data.size());
-      _data.swap(new_data);
+      VertexDataBuffer new_buffer(_buffer.get_size());
+      array_data->reverse_data_endianness(new_buffer.get_write_pointer(), _buffer.get_read_pointer(), _buffer.get_size());
+      _buffer.swap(new_buffer);
     }
   }
 
-  _data_full_size = _data.size();
-  array_data->set_lru_size(_data_full_size);
+  array_data->set_lru_size(_buffer.get_size());
 
   _modified = Geom::get_next_modified();
 }
@@ -927,23 +639,22 @@ set_num_rows(int n) {
   check_resident();
 
   int stride = _object->_array_format->get_stride();
-  int delta = n - (_cdata->_data.size() / stride);
-  
-  if (delta != 0) {
-    if (delta > 0) {
-      _cdata->_data.insert(_cdata->_data.end(), delta * stride, 0);
-      
-    } else {
-      _cdata->_data.erase(_cdata->_data.begin() + n * stride, 
-                          _cdata->_data.end());
+  size_t new_size = n * stride;
+  size_t orig_size = _cdata->_buffer.get_size();
+
+  if (new_size != orig_size) {
+    _cdata->_buffer.clean_realloc(new_size);
+
+    // Now ensure that the newly-added rows are initialized to 0.
+    if (new_size > orig_size) {
+      memset(_cdata->_buffer.get_write_pointer() + orig_size, 0,
+             new_size - orig_size);
     }
 
     _cdata->_modified = Geom::get_next_modified();
-    _cdata->_data_full_size = _cdata->_data.size();
 
     if (get_current_thread()->get_pipeline_stage() == 0) {
-      _object->set_ram_class(RC_resident);
-      _object->set_lru_size(_cdata->_data_full_size);
+      _object->set_lru_size(_cdata->_buffer.get_size());
     }
     return true;
   }
@@ -962,19 +673,18 @@ unclean_set_num_rows(int n) {
   check_resident();
 
   int stride = _object->_array_format->get_stride();
-  int delta = n - (_cdata->_data.size() / stride);
-  
-  if (delta != 0) {
-    // Just make a new array.  No reason to keep the old one around.
-    GeomVertexArrayData::Data new_data(n * stride, GeomVertexArrayData::get_class_type());
-    _cdata->_data.swap(new_data);
+  size_t new_size = n * stride;
+  size_t orig_size = _cdata->_buffer.get_size();
+
+  if (new_size != orig_size) {
+    _cdata->_buffer.unclean_realloc(new_size);
+    // No need to fill to zero or copy the old buffer, since this is
+    // unclean_set_num_rows().
 
     _cdata->_modified = Geom::get_next_modified();
-    _cdata->_data_full_size = _cdata->_data.size();
 
     if (get_current_thread()->get_pipeline_stage() == 0) {
-      _object->set_ram_class(RC_resident);
-      _object->set_lru_size(_cdata->_data_full_size);
+      _object->set_lru_size(_cdata->_buffer.get_size());
     }
     return true;
   }
@@ -993,13 +703,15 @@ copy_data_from(const GeomVertexArrayDataHandle *other) {
   check_resident();
   other->check_resident();
 
-  _cdata->_data = other->_cdata->_data;
+  _cdata->_buffer.unclean_realloc(other->_cdata->_buffer.get_size());
+  memcpy(_cdata->_buffer.get_write_pointer(),
+         other->_cdata->_buffer.get_read_pointer(),
+         other->_cdata->_buffer.get_size());
+
   _cdata->_modified = Geom::get_next_modified();
-  _cdata->_data_full_size = _cdata->_data.size();
 
   if (get_current_thread()->get_pipeline_stage() == 0) {
-    _object->set_ram_class(RC_resident);
-    _object->set_lru_size(_cdata->_data_full_size);
+    _object->set_lru_size(_cdata->_buffer.get_size());
   }
 }
 
@@ -1019,30 +731,107 @@ copy_subdata_from(size_t to_start, size_t to_size,
   check_resident();
   other->check_resident();
 
-  GeomVertexArrayData::Data &to_v = _cdata->_data;
-  to_start = min(to_start, to_v.size());
-  to_size = min(to_size, to_v.size() - to_start);
+  VertexDataBuffer &to_buffer = _cdata->_buffer;
+  to_start = min(to_start, to_buffer.get_size());
+  to_size = min(to_size, to_buffer.get_size() - to_start);
 
-  from_start = min(from_start, other->_cdata->_data.size());
-  from_size = min(from_size, other->_cdata->_data.size() - from_start);
+  const VertexDataBuffer &from_buffer = other->_cdata->_buffer;
+  from_start = min(from_start, from_buffer.get_size());
+  from_size = min(from_size, from_buffer.get_size() - from_start);
 
   if (from_size < to_size) {
     // Reduce the array.
-    to_v.erase(to_v.begin() + to_start + from_size, to_v.begin() + to_start + to_size);
+    unsigned char *pointer = to_buffer.get_write_pointer();
+    memmove(pointer + to_start + to_size, 
+            pointer + to_start + from_size,
+            from_size - (to_start + to_size));
+    to_buffer.clean_realloc(to_buffer.get_size() + from_size - to_size);
 
   } else if (to_size < from_size) {
     // Expand the array.
-    to_v.insert(to_v.begin() + to_start + to_size, from_size - to_size, char());
+    to_buffer.clean_realloc(to_buffer.get_size() + from_size - to_size);
+    unsigned char *pointer = to_buffer.get_write_pointer();
+    memmove(pointer + to_start + to_size, 
+            pointer + to_start + from_size,
+            from_size - (to_start + from_size));
   }
 
   // Now copy the data.
-  memcpy(&to_v[0] + to_start, other->get_pointer() + from_start, from_size);
+  memcpy(to_buffer.get_write_pointer() + to_start, 
+         other->get_read_pointer() + from_start, 
+         from_size);
+  _cdata->_modified = Geom::get_next_modified();
+
+  if (get_current_thread()->get_pipeline_stage() == 0) {
+    _object->set_lru_size(_cdata->_buffer.get_size());
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: GeomVertexArrayDataHandle::set_data
+//       Access: Public
+//  Description: Replaces the entire raw data array with the contents
+//               of the indicated string.  This is primarily for the
+//               benefit of high-level languages like Python.
+////////////////////////////////////////////////////////////////////
+void GeomVertexArrayDataHandle::
+set_data(const string &data) {
+  nassertv(_writable);
+  check_resident();
+
+  _cdata->_buffer.unclean_realloc(data.size());
+  memcpy(_cdata->_buffer.get_write_pointer(), data.data(), data.size());
+
   _cdata->_modified = Geom::get_next_modified();
-  _cdata->_data_full_size = _cdata->_data.size();
 
   if (get_current_thread()->get_pipeline_stage() == 0) {
-    _object->set_ram_class(RC_resident);
-    _object->set_lru_size(_cdata->_data_full_size);
+    _object->set_lru_size(_cdata->_buffer.get_size());
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: GeomVertexArrayDataHandle::copy_subdata_from
+//       Access: Public
+//  Description: Replaces a portion of the data array from the
+//               indicated string.  If size != data.size(), the size
+//               of this data array is adjusted accordingly.
+//
+//               This is primarily for the benefit of high-level
+//               languages like Python.
+////////////////////////////////////////////////////////////////////
+void GeomVertexArrayDataHandle::
+set_subdata(size_t start, size_t size, const string &data) {
+  nassertv(_writable);
+  check_resident();
+
+  VertexDataBuffer &to_buffer = _cdata->_buffer;
+  start = min(start, to_buffer.get_size());
+  size = min(size, to_buffer.get_size() - start);
+  
+  size_t from_size = data.size();
+
+  if (from_size < size) {
+    // Reduce the array.
+    unsigned char *pointer = to_buffer.get_write_pointer();
+    memmove(pointer + start + size, 
+            pointer + start + from_size,
+            from_size - (start + size));
+    to_buffer.clean_realloc(to_buffer.get_size() + from_size - size);
+
+  } else if (size < from_size) {
+    // Expand the array.
+    to_buffer.clean_realloc(to_buffer.get_size() + from_size - size);
+    unsigned char *pointer = to_buffer.get_write_pointer();
+    memmove(pointer + start + size, 
+            pointer + start + from_size,
+            from_size - (start + from_size));
+  }
+
+  // Now copy the data.
+  memcpy(to_buffer.get_write_pointer() + start, data.data(), from_size);
+  _cdata->_modified = Geom::get_next_modified();
+
+  if (get_current_thread()->get_pipeline_stage() == 0) {
+    _object->set_lru_size(_cdata->_buffer.get_size());
+  }
+}

panda/src/gobj/geomVertexArrayData.h

@@ -35,12 +35,14 @@
 #include "pmap.h"
 #include "reMutex.h"
 #include "simpleLru.h"
+#include "vertexDataBuffer.h"
+#include "config_gobj.h"
 
 class PreparedGraphicsObjects;
 class VertexBufferContext;
 class GraphicsStateGuardianBase;
 class GeomVertexArrayDataHandle;
-class VertexDataSaveFile;
+class VertexDataBook;
 class SimpleAllocatorBlock;
 
 ////////////////////////////////////////////////////////////////////
@@ -106,28 +108,21 @@ PUBLISHED:
   bool release(PreparedGraphicsObjects *prepared_objects);
   int release_all();
 
-  INLINE RamClass get_ram_class() const;
-
-  INLINE static SimpleLru *get_global_lru(RamClass rclass);
+  INLINE static SimpleLru *get_independent_lru();
+  INLINE static SimpleLru *get_small_lru();
   static void lru_epoch();
-  INLINE static VertexDataSaveFile *get_save_file();
-
-  void make_resident();
-  void make_compressed();
-  void make_disk();
-  void restore_from_disk();
+  INLINE static VertexDataBook &get_book();
 
 public:
   virtual void evict_lru();
 
 private:
+  INLINE void set_lru_size(size_t lru_size);
+
   void clear_prepared(PreparedGraphicsObjects *prepared_objects);
   void reverse_data_endianness(unsigned char *dest, 
                                const unsigned char *source, size_t size);
 
-  INLINE void set_ram_class(RamClass rclass);
-  static void make_save_file();
-
 
   CPT(GeomVertexArrayFormat) _array_format;
 
@@ -143,11 +138,6 @@ private:
   // to indicate the data must be endian-reversed in finalize().
   bool _endian_reversed;
 
-  RamClass _ram_class;
-  SimpleAllocatorBlock *_saved_block;
-
-  typedef pvector<unsigned char> Data;
-
   // This is the data that must be cycled between pipeline stages.
   class EXPCL_PANDA CData : public CycleData {
   public:
@@ -167,8 +157,7 @@ private:
     }
 
     UsageHint _usage_hint;
-    Data _data;
-    size_t _data_full_size;
+    VertexDataBuffer _buffer;
     UpdateSeq _modified;
 
     // This implements read-write locking.  Anyone who gets the data for
@@ -195,16 +184,9 @@ private:
   typedef CycleDataStageReader<CData> CDStageReader;
   typedef CycleDataStageWriter<CData> CDStageWriter;
 
-  static SimpleLru _ram_lru;
-  static SimpleLru _compressed_lru;
-  static SimpleLru _disk_lru;
-  static SimpleLru *_global_lru[RC_end_of_list];
-  static VertexDataSaveFile *_save_file;
-
-  static PStatCollector _vdata_compress_pcollector;
-  static PStatCollector _vdata_decompress_pcollector;
-  static PStatCollector _vdata_save_pcollector;
-  static PStatCollector _vdata_restore_pcollector;
+  static SimpleLru _independent_lru;
+  static SimpleLru _small_lru;
+  static VertexDataBook _book;
 
 public:
   static void register_with_read_factory();
@@ -277,8 +259,8 @@ public:
   INLINE const GeomVertexArrayData *get_object() const;
   INLINE GeomVertexArrayData *get_object();
 
-  INLINE const unsigned char *get_pointer() const;
-  INLINE unsigned char *get_pointer();
+  INLINE const unsigned char *get_read_pointer() const;
+  INLINE unsigned char *get_write_pointer();
 
 PUBLISHED:
   INLINE const GeomVertexArrayFormat *get_array_format() const;
@@ -297,12 +279,10 @@ PUBLISHED:
                          const GeomVertexArrayDataHandle *other,
                          size_t from_start, size_t from_size);
 
-  /*
   INLINE string get_data() const;
   void set_data(const string &data);
   INLINE string get_subdata(size_t start, size_t size) const;
   void set_subdata(size_t start, size_t size, const string &data);
-  */
 
   INLINE void check_resident() const;
   

panda/src/gobj/geomVertexData.cxx

@@ -489,7 +489,7 @@ copy_from(const GeomVertexData *source, bool keep_data_objects,
   for (source_i = 0; source_i < num_arrays; ++source_i) {
     CPT(GeomVertexArrayData) array_obj = source->get_array(source_i);
     CPT(GeomVertexArrayDataHandle) array_handle = array_obj->get_handle();
-    const unsigned char *array_data = array_handle->get_pointer();
+    const unsigned char *array_data = array_handle->get_read_pointer();
     const GeomVertexArrayFormat *source_array_format = source_format->get_array(source_i);
     int num_columns = source_array_format->get_num_columns();
     for (int di = 0; di < num_columns; ++di) {
@@ -508,7 +508,7 @@ copy_from(const GeomVertexData *source, bool keep_data_objects,
           // We can do a quick bytewise copy.
           PT(GeomVertexArrayData) dest_array_obj = modify_array(dest_i);
           PT(GeomVertexArrayDataHandle) dest_handle = dest_array_obj->modify_handle();
-          unsigned char *dest_array_data = dest_handle->get_pointer();
+          unsigned char *dest_array_data = dest_handle->get_write_pointer();
 
           bytewise_copy(dest_array_data + dest_column->get_start(), 
                         dest_array_format->get_stride(),
@@ -520,7 +520,7 @@ copy_from(const GeomVertexData *source, bool keep_data_objects,
           // A common special case: OpenGL color to DirectX color.
           PT(GeomVertexArrayData) dest_array_obj = modify_array(dest_i);
           PT(GeomVertexArrayDataHandle) dest_handle = dest_array_obj->modify_handle();
-          unsigned char *dest_array_data = dest_handle->get_pointer();
+          unsigned char *dest_array_data = dest_handle->get_write_pointer();
 
           uint8_rgba_to_packed_argb
             (dest_array_data + dest_column->get_start(), 
@@ -534,7 +534,7 @@ copy_from(const GeomVertexData *source, bool keep_data_objects,
           // color.
           PT(GeomVertexArrayData) dest_array_obj = modify_array(dest_i);
           PT(GeomVertexArrayDataHandle) dest_handle = dest_array_obj->modify_handle();
-          unsigned char *dest_array_data = dest_handle->get_pointer();
+          unsigned char *dest_array_data = dest_handle->get_write_pointer();
 
           packed_argb_to_uint8_rgba
             (dest_array_data + dest_column->get_start(), 
@@ -656,11 +656,11 @@ copy_row_from(int dest_row, const GeomVertexData *source,
   for (int i = 0; i < num_arrays; ++i) {
     PT(GeomVertexArrayData) dest_array_obj = modify_array(i);
     PT(GeomVertexArrayDataHandle) dest_handle = dest_array_obj->modify_handle();
-    unsigned char *dest_array_data = dest_handle->get_pointer();
+    unsigned char *dest_array_data = dest_handle->get_write_pointer();
 
     CPT(GeomVertexArrayData) source_array_obj = source->get_array(i);
     CPT(GeomVertexArrayDataHandle) source_array_handle = source_array_obj->get_handle();
-    const unsigned char *source_array_data = source_array_handle->get_pointer();
+    const unsigned char *source_array_data = source_array_handle->get_read_pointer();
 
     const GeomVertexArrayFormat *array_format = source_format->get_array(i);
     int stride = array_format->get_stride();
@@ -1897,7 +1897,7 @@ set_num_rows(int n) {
       array_format->get_column(InternalName::get_color());
     int stride = array_format->get_stride();
     unsigned char *start = 
-      array_writer->get_pointer() + column->get_start();
+      array_writer->get_write_pointer() + column->get_start();
     unsigned char *stop = start + array_writer->get_data_size_bytes();
     unsigned char *pointer = start + stride * orig_color_rows;
     int num_values = column->get_num_values();

panda/src/gobj/geomVertexFormat.cxx

@@ -576,7 +576,7 @@ write_with_data(ostream &out, int indent_level,
     << data->get_num_rows() << " rows.\n";
   for (size_t i = 0; i < _arrays.size(); i++) {
     CPT(GeomVertexArrayDataHandle) handle = data->get_array(i)->get_handle();
-    const unsigned char *array_data = handle->get_pointer();
+    const unsigned char *array_data = handle->get_read_pointer();
     indent(out, indent_level)
       << "Array " << i << " (" << (void *)array_data << ", "
       << *_arrays[i] << "):\n";

panda/src/gobj/geomVertexReader.I

@@ -508,7 +508,7 @@ get_packer() const {
 ////////////////////////////////////////////////////////////////////
 INLINE void GeomVertexReader::
 set_pointer(int row) {
-  _pointer_begin = _handle->get_pointer();
+  _pointer_begin = _handle->get_read_pointer();
   _pointer_end = _pointer_begin + _handle->get_data_size_bytes();
   quick_set_pointer(row);
 }
@@ -526,7 +526,7 @@ quick_set_pointer(int row) {
 
 #if defined(_DEBUG)
   // Make sure we still have the same pointer as stored in the array.
-  nassertv(_pointer_begin == _handle->get_pointer());
+  nassertv(_pointer_begin == _handle->get_read_pointer());
 #endif
 
   _pointer = _pointer_begin + _packer->_column->get_start() + _stride * row;
@@ -547,7 +547,7 @@ inc_pointer() {
 #if defined(_DEBUG)
   nassertr(_pointer < _pointer_end, empty_buffer);
   // Make sure we still have the same pointer as stored in the array.
-  nassertr(_pointer_begin == _handle->get_pointer(), empty_buffer);
+  nassertr(_pointer_begin == _handle->get_read_pointer(), empty_buffer);
   nassertr(_pointer < _pointer_begin + _handle->get_data_size_bytes(), empty_buffer);
 #endif
 

panda/src/gobj/geomVertexWriter.I

@@ -811,7 +811,7 @@ get_packer() const {
 ////////////////////////////////////////////////////////////////////
 INLINE void GeomVertexWriter::
 set_pointer(int row) {
-  _pointer_begin = _handle->get_pointer();
+  _pointer_begin = _handle->get_write_pointer();
   _pointer_end = _pointer_begin + _handle->get_data_size_bytes();
   quick_set_pointer(row);
 }
@@ -829,7 +829,7 @@ quick_set_pointer(int row) {
 
 #if defined(_DEBUG)
   // Make sure we still have the same pointer as stored in the array.
-  nassertv(_pointer_begin == _handle->get_pointer());
+  nassertv(_pointer_begin == _handle->get_write_pointer());
 #endif
 
   _pointer = _pointer_begin + _packer->_column->get_start() + _stride * row;
@@ -850,7 +850,7 @@ inc_pointer() {
 #if defined(_DEBUG)
   nassertr(_pointer < _pointer_end, empty_buffer);
   // Make sure we still have the same pointer as stored in the array.
-  nassertr(_pointer_begin == _handle->get_pointer(), empty_buffer);
+  nassertr(_pointer_begin == _handle->get_write_pointer(), empty_buffer);
   nassertr(_pointer < _pointer_begin + _handle->get_data_size_bytes(), empty_buffer);
 #endif
 

panda/src/gobj/gobj_composite2.cxx

@@ -23,6 +23,8 @@
 #include "userVertexSlider.cxx"
 #include "userVertexTransform.cxx"
 #include "vertexBufferContext.cxx"
+#include "vertexDataBook.cxx"
+#include "vertexDataBuffer.cxx"
 #include "vertexDataSaveFile.cxx"
 #include "vertexSlider.cxx"
 #include "vertexTransform.cxx"

panda/src/gobj/simpleAllocator.I

@@ -26,7 +26,8 @@ INLINE SimpleAllocator::
 SimpleAllocator(size_t max_size) : 
   LinkedListNode(true),
   _total_size(0),
-  _max_size(max_size)
+  _max_size(max_size),
+  _contiguous(max_size)
 {
 }
 
@@ -62,6 +63,58 @@ set_max_size(size_t max_size) {
   _max_size = max_size;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocator::get_contiguous
+//       Access: Published
+//  Description: Returns an upper-bound estimate of the size of the
+//               largest contiguous block that may be allocated.  It
+//               is guaranteed that an attempt to allocate a block
+//               larger than this will fail, though it is not
+//               guaranteed that an attempt to allocate a block this
+//               size or smaller will succeed.
+////////////////////////////////////////////////////////////////////
+INLINE size_t SimpleAllocator::
+get_contiguous() const {
+  return _contiguous;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocator::get_first_block
+//       Access: Published
+//  Description: Returns a pointer to the first allocated block, or
+//               NULL if there are no allocated blocks.
+////////////////////////////////////////////////////////////////////
+INLINE SimpleAllocatorBlock *SimpleAllocator::
+get_first_block() const {
+  return (_next == this) ? (SimpleAllocatorBlock *)NULL : (SimpleAllocatorBlock *)_next;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocator::increase_contiguous
+//       Access: Protected
+//  Description: Some space has been made available following the
+//               indicated block.  Increase the contiguous space
+//               accordingly.
+////////////////////////////////////////////////////////////////////
+INLINE void SimpleAllocator::
+mark_contiguous(const LinkedListNode *block) {
+  size_t space;
+  if (block == this) {
+    // This is the beginning of the list.
+    if (_next == this) {
+      // And the list is empty.
+      space = _max_size;
+    } else {
+      space = ((SimpleAllocatorBlock *)_next)->get_start();
+    }
+  } else {
+    space = ((SimpleAllocatorBlock *)block)->get_max_size() - ((SimpleAllocatorBlock *)block)->get_size();
+  }
+  if (space > _contiguous) {
+    _contiguous = space;
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: SimpleAllocatorBlock::Constructor
 //       Access: Private
@@ -97,7 +150,9 @@ INLINE void SimpleAllocatorBlock::
 free() {
   if (_allocator != (SimpleAllocator *)NULL) {
     _allocator->_total_size -= _size;
+    LinkedListNode *prev = _prev;
     remove_from_list();
+    _allocator->mark_contiguous(prev);
     _allocator = NULL;
   }
 }
@@ -147,3 +202,60 @@ INLINE bool SimpleAllocatorBlock::
 is_free() const {
   return (_allocator != (SimpleAllocator *)NULL);
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocatorBlock::get_max_size
+//       Access: Published
+//  Description: Returns the maximum size this block can be
+//               reallocated to, as limited by the following block.
+////////////////////////////////////////////////////////////////////
+INLINE size_t SimpleAllocatorBlock::
+get_max_size() const {
+  nassertr(_allocator != (SimpleAllocator *)NULL, 0);
+
+  size_t end;
+  if (_next == _allocator) {
+    end = _allocator->get_max_size();
+  } else {
+    end = ((SimpleAllocatorBlock *)_next)->_start;
+  }
+  return end - _start;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocatorBlock::realloc
+//       Access: Published
+//  Description: Changes the size of this block to the specified size.
+//               Returns true if the change is accepted, false if
+//               there was not enough room.
+////////////////////////////////////////////////////////////////////
+INLINE bool SimpleAllocatorBlock::
+realloc(size_t size) {
+  if (size > get_max_size()) {
+    return false;
+  }
+
+  _allocator->_total_size -= _size;
+  _allocator->_total_size += size;
+
+  if (size < _size) {
+    // We're decreasing the block size.
+    _size = size;
+    _allocator->mark_contiguous(this);
+  } else {
+    // We're increasing the block size.
+    _size = size;
+  }
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocatorBlock::get_next_block
+//       Access: Published
+//  Description: Returns a pointer to the next allocated block in the
+//               chain, or NULL if there are no more allocated blocks.
+////////////////////////////////////////////////////////////////////
+INLINE SimpleAllocatorBlock *SimpleAllocatorBlock::
+get_next_block() const {
+  return (_next == _allocator) ? (SimpleAllocatorBlock *)NULL : (SimpleAllocatorBlock *)_next;
+}

panda/src/gobj/simpleAllocator.cxx

@@ -20,7 +20,7 @@
 
 ////////////////////////////////////////////////////////////////////
 //     Function: SimpleAllocator::Destructor
-//       Access: Published
+//       Access: Published, Virtual
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 SimpleAllocator::
@@ -43,12 +43,17 @@ SimpleAllocator::
 ////////////////////////////////////////////////////////////////////
 SimpleAllocatorBlock *SimpleAllocator::
 alloc(size_t size) {
+  if (size > _contiguous) {
+    // Don't even bother.
+    return NULL;
+  }
+
   // First fit algorithm: walk through all the empty blocks until we
   // find one that has enough room.
 
   SimpleAllocatorBlock *block = NULL;
   size_t end = 0;
-
+  size_t best = 0;
   if (_next != this) {
     // We have at least one allocated block.
     block = (SimpleAllocatorBlock *)_next;
@@ -59,11 +64,14 @@ alloc(size_t size) {
       SimpleAllocatorBlock *next = (SimpleAllocatorBlock *)block->_next;
       size_t free_size = next->_start - end;
       if (size <= free_size) {
-        SimpleAllocatorBlock *new_block = new SimpleAllocatorBlock(this, end, size);
+        SimpleAllocatorBlock *new_block = make_block(end, size);
         new_block->insert_before(next);
         _total_size += size;
         return new_block;
       }
+      if (free_size > best) {
+        best = free_size;
+      }
       
       block = next;
       end = block->_start + block->_size;
@@ -73,12 +81,20 @@ alloc(size_t size) {
   // No free blocks; check for room at the end.
   size_t free_size = _max_size - end;
   if (size <= free_size) {
-    SimpleAllocatorBlock *new_block = new SimpleAllocatorBlock(this, end, size);
+    SimpleAllocatorBlock *new_block = make_block(end, size);
     new_block->insert_before(this);
     _total_size += size;
     return new_block;
   }
 
+  if (free_size > best) {
+    best = free_size;
+  }
+
+  // Now that we've walked through the entire list of blocks, we
+  // really do know accurately what the largest contiguous block is.
+  _contiguous = best;
+
   // No room for this block.
   return NULL;
 }
@@ -94,6 +110,20 @@ output(ostream &out) const {
       << " allocated";
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: SimpleAllocatorBlock::output
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+void SimpleAllocatorBlock::
+output(ostream &out) const {
+  if (_allocator == (SimpleAllocator *)NULL) {
+    out << "free block\n";
+  } else {
+    out << "block of size " << _size << " at " << _start;
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: SimpleAllocator::write
 //       Access: Published
@@ -112,16 +142,15 @@ write(ostream &out) const {
   }
 }
 
+
 ////////////////////////////////////////////////////////////////////
-//     Function: SimpleAllocatorBlock::output
-//       Access: Published
-//  Description: 
+//     Function: SimpleAllocator::make_block
+//       Access: Protected, Virtual
+//  Description: Creates a new SimpleAllocatorBlock object.  Override
+//               this function to specialize the block type returned.
 ////////////////////////////////////////////////////////////////////
-void SimpleAllocatorBlock::
-output(ostream &out) const {
-  if (_allocator == (SimpleAllocator *)NULL) {
-    out << "free block\n";
-  } else {
-    out << "block of size " << _size << " at " << _start;
-  }
+SimpleAllocatorBlock *SimpleAllocator::
+make_block(size_t start, size_t size) {
+  return new SimpleAllocatorBlock(this, start, size);
 }
+

panda/src/gobj/simpleAllocator.h

@@ -35,17 +35,24 @@ class SimpleAllocatorBlock;
 class EXPCL_PANDA SimpleAllocator : public LinkedListNode {
 PUBLISHED:
   INLINE SimpleAllocator(size_t max_size);
-  ~SimpleAllocator();
+  virtual ~SimpleAllocator();
 
   SimpleAllocatorBlock *alloc(size_t size);
 
   INLINE size_t get_total_size() const;
   INLINE size_t get_max_size() const;
   INLINE void set_max_size(size_t max_size);
+  INLINE size_t get_contiguous() const;
+
+  INLINE SimpleAllocatorBlock *get_first_block() const;
 
   void output(ostream &out) const;
   void write(ostream &out) const;
 
+protected:
+  virtual SimpleAllocatorBlock *make_block(size_t start, size_t size);
+  INLINE void mark_contiguous(const LinkedListNode *block);
+
 private:
   // This is implemented as a linked-list chain of allocated blocks.
   // Free blocks are implicit.  Blocks are kept in sorted order from
@@ -59,6 +66,11 @@ private:
   size_t _total_size;
   size_t _max_size;
 
+  // This is what we currently believe our max contiguous space to be.
+  // This guess might be larger than the actual available space, but
+  // it will not be smaller.
+  size_t _contiguous;
+
   friend class SimpleAllocatorBlock;
 };
 
@@ -68,7 +80,7 @@ private:
 //               SimpleAllocator::alloc().
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDA SimpleAllocatorBlock : public LinkedListNode {
-private:
+protected:
   INLINE SimpleAllocatorBlock(SimpleAllocator *alloc,
                               size_t start, size_t size);
 
@@ -82,6 +94,11 @@ PUBLISHED:
   INLINE size_t get_size() const;
   INLINE bool is_free() const;
 
+  INLINE size_t get_max_size() const;
+  INLINE bool realloc(size_t size);
+
+  INLINE SimpleAllocatorBlock *get_next_block() const;
+
   void output(ostream &out) const;
 
 private:

panda/src/gobj/simpleLru.I

@@ -161,11 +161,14 @@ dequeue_lru() {
 //       Access: Published
 //  Description: To be called when the page is used; this will move it
 //               to the tail of the SimpleLru queue it is already on.
+//
+//               This method is const because it's not technically
+//               modifying the contents of the page itself.
 ////////////////////////////////////////////////////////////////////
 INLINE void SimpleLruPage::
-mark_used_lru() {
+mark_used_lru() const {
   if (_lru != (SimpleLru *)NULL) {
-    mark_used_lru(_lru);
+    ((SimpleLruPage *)this)->mark_used_lru(_lru);
   }
 }
 
@@ -177,7 +180,6 @@ mark_used_lru() {
 ////////////////////////////////////////////////////////////////////
 INLINE void SimpleLruPage::
 mark_used_lru(SimpleLru *lru) {
-  dequeue_lru();
   enqueue_lru(lru);
 }
 

panda/src/gobj/simpleLru.h

@@ -70,7 +70,7 @@ PUBLISHED:
   INLINE void enqueue_lru(SimpleLru *lru);
   INLINE void dequeue_lru();
 
-  INLINE void mark_used_lru();
+  INLINE void mark_used_lru() const;
   INLINE void mark_used_lru(SimpleLru *lru);
 
   INLINE size_t get_lru_size() const;

panda/src/gobj/vertexDataBook.I

@@ -0,0 +1,212 @@
+// Filename: vertexDataBook.I
+// Created by:  drose (16May07)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001 - 2004, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://etc.cmu.edu/panda3d/docs/license/ .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBook::get_num_pages
+//       Access: Published
+//  Description: Returns the number of pages created for the book.
+////////////////////////////////////////////////////////////////////
+INLINE int VertexDataBook::
+get_num_pages() const {
+  return _pages.size();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBook::get_page
+//       Access: Published
+//  Description: Returns the nth page created for the book.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataPage *VertexDataBook::
+get_page(int n) const {
+  nassertr(n >= 0 && n < (int)_pages.size(), NULL);
+  return _pages[n];
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBook::create_new_page
+//       Access: Private
+//  Description: Creates a new page of sufficient size to hold the
+//               requested block.  The page is not added to the _pages
+//               list.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataPage *VertexDataBook::
+create_new_page(size_t size) {
+  size_t page_size = ((size + _block_size - 1) / _block_size) * _block_size;
+  return new VertexDataPage(page_size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::get_ram_class
+//       Access: Published
+//  Description: Returns the current ram class of the array.  If this
+//               is other than RC_resident, the array data is not
+//               resident in memory.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataPage::RamClass VertexDataPage::
+get_ram_class() const {
+  return _ram_class;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::check_resident
+//       Access: Published
+//  Description: Forces the vertex data into system RAM, if it is not
+//               already there; also, marks it recently-used.
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataPage::
+check_resident() const {
+  if (get_ram_class() != RC_resident) {
+    ((VertexDataPage *)this)->make_resident();
+  } else {
+    ((VertexDataPage *)this)->mark_used_lru();
+  }
+  nassertv(_size == _uncompressed_size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::alloc
+//       Access: Published
+//  Description: Allocates a new block.  Returns NULL if a block of the
+//               requested size cannot be allocated.
+//
+//               To free the allocated block, call block->free(), or
+//               simply delete the block pointer.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBlock *VertexDataPage::
+alloc(size_t size) {
+  check_resident();
+  return (VertexDataBlock *)SimpleAllocator::alloc(size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::get_first_block
+//       Access: Published
+//  Description: Returns a pointer to the first allocated block, or
+//               NULL if there are no allocated blocks.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBlock *VertexDataPage::
+get_first_block() const {
+  check_resident();
+  return (VertexDataBlock *)SimpleAllocator::get_first_block();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::get_total_page_size
+//       Access: Published, Static
+//  Description: Returns the byte count allocated to all
+//               VertexDataPages currently in existance.
+////////////////////////////////////////////////////////////////////
+INLINE size_t VertexDataPage::
+get_total_page_size() {
+  return _total_page_size;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::get_global_lru
+//       Access: Published, Static
+//  Description: Returns a pointer to the global LRU object that
+//               manages the VertexDataPage's with the indicated
+//               RamClass.
+////////////////////////////////////////////////////////////////////
+INLINE SimpleLru *VertexDataPage::
+get_global_lru(RamClass rclass) {
+  nassertr(rclass >= 0 && rclass < RC_end_of_list, NULL);
+  return _global_lru[rclass];
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::get_save_file
+//       Access: Published, Static
+//  Description: Returns the global VertexDataSaveFile that will be
+//               used to save vertex data buffers to disk when
+//               necessary.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataSaveFile *VertexDataPage::
+get_save_file() {
+  if (_save_file == (VertexDataSaveFile *)NULL) {
+    make_save_file();
+  }
+  return _save_file;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::get_page_data
+//       Access: Public
+//  Description: Returns a pointer to the page's data area.
+////////////////////////////////////////////////////////////////////
+INLINE unsigned char *VertexDataPage::
+get_page_data() const {
+  check_resident();
+  return _page_data;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::set_ram_class
+//       Access: Private
+//  Description: Puts the data in a new ram class.
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataPage::
+set_ram_class(RamClass rclass) {
+  _ram_class = rclass;
+  mark_used_lru(_global_lru[rclass]);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBlock::Constructor
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBlock::
+VertexDataBlock(VertexDataPage *page, size_t start, size_t size) :
+  SimpleAllocatorBlock(page, start, size)
+{
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBlock::get_page
+//       Access: Published
+//  Description: Returns the page from which this buffer was
+//               allocated.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataPage *VertexDataBlock::
+get_page() const {
+  return (VertexDataPage *)get_allocator();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBlock::get_pointer
+//       Access: Public
+//  Description: Returns a pointer to the start of the allocated
+//               memory for this buffer.
+////////////////////////////////////////////////////////////////////
+INLINE unsigned char *VertexDataBlock::
+get_pointer() const {
+  return get_page()->get_page_data() + get_start();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBlock::get_next_block
+//       Access: Published
+//  Description: Returns a pointer to the next allocated block in the
+//               chain, or NULL if there are no more allocated blocks.
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBlock *VertexDataBlock::
+get_next_block() const {
+  return (VertexDataBlock *)SimpleAllocatorBlock::get_next_block();
+}

panda/src/gobj/vertexDataBook.cxx

@@ -0,0 +1,455 @@
+// Filename: vertexDataBook.cxx
+// Created by:  drose (16May07)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001 - 2004, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://etc.cmu.edu/panda3d/docs/license/ .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "vertexDataBook.h"
+#include "configVariableInt.h"
+#include "vertexDataSaveFile.h"
+#include "pStatTimer.h"
+
+#ifdef HAVE_ZLIB
+#include <zlib.h>
+#endif
+
+ConfigVariableInt max_resident_vertex_data
+("max-resident-vertex-data", -1,
+ PRC_DESC("Specifies the maximum number of bytes of all vertex data "
+          "that is allowed to remain resident in system RAM at one time. "
+          "If more than this number of bytes of vertices are created, "
+          "the least-recently-used ones will be temporarily compressed in "
+          "system RAM until they are needed.  Set it to -1 for no limit."));
+
+ConfigVariableInt max_compressed_vertex_data
+("max-compressed-vertex-data", 0,
+ PRC_DESC("Specifies the maximum number of bytes of all vertex data "
+          "that is allowed to remain compressed in system RAM at one time. "
+          "If more than this number of bytes of vertices are created, "
+          "the least-recently-used ones will be temporarily flushed to "
+          "disk until they are needed.  Set it to -1 for no limit."));
+
+ConfigVariableInt vertex_data_compression_level
+("vertex-data-compression-level", 1,
+ PRC_DESC("Specifies the zlib compression level to use when compressing "
+          "vertex data.  The number should be in the range 1 to 9, where "
+          "larger values are slower but give better compression."));
+
+ConfigVariableInt max_disk_vertex_data
+("max-disk-vertex-data", -1,
+ PRC_DESC("Specifies the maximum number of bytes of vertex data "
+          "that is allowed to be written to disk.  Set it to -1 for no "
+          "limit."));
+
+SimpleLru VertexDataPage::_resident_lru(max_resident_vertex_data);
+SimpleLru VertexDataPage::_compressed_lru(max_compressed_vertex_data);
+SimpleLru VertexDataPage::_disk_lru(0);
+
+SimpleLru *VertexDataPage::_global_lru[RC_end_of_list] = {
+  &VertexDataPage::_resident_lru,
+  &VertexDataPage::_compressed_lru,
+  &VertexDataPage::_disk_lru,
+  &VertexDataPage::_disk_lru,
+};
+
+size_t VertexDataPage::_total_page_size = 0;
+VertexDataSaveFile *VertexDataPage::_save_file;
+
+PStatCollector VertexDataPage::_vdata_compress_pcollector("*:Vertex Data:Compress");
+PStatCollector VertexDataPage::_vdata_decompress_pcollector("*:Vertex Data:Decompress");
+PStatCollector VertexDataPage::_vdata_save_pcollector("*:Vertex Data:Save");
+PStatCollector VertexDataPage::_vdata_restore_pcollector("*:Vertex Data:Restore");
+
+TypeHandle VertexDataPage::_type_handle;
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBook::Constructor
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VertexDataBook::
+VertexDataBook(size_t block_size) : _block_size(block_size) {
+  _next_pi = 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBook::Destructor
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VertexDataBook::
+~VertexDataBook() {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBook::alloc
+//       Access: Published
+//  Description: Allocates and returns a new VertexDataBuffer of the
+//               requested size.
+////////////////////////////////////////////////////////////////////
+VertexDataBlock *VertexDataBook::
+alloc(size_t size) {
+  // First, try to allocate from the last page that worked; then
+  // continue to the end of the list.
+  size_t pi = _next_pi;
+  while (pi < _pages.size()) {
+    VertexDataBlock *block = _pages[pi]->alloc(size);
+    if (block != (VertexDataBlock *)NULL) {
+      _next_pi = pi;
+      return block;
+    }
+    if (_pages[pi]->get_total_size() == 0) {
+      // This page is empty, but must have been too small.  Create a
+      // new page in its place.
+      delete _pages[pi];
+      _pages[pi] = create_new_page(size);
+      return _pages[pi]->alloc(size);
+    }
+    ++pi;
+  }
+
+  // Then, go back to the beginning and try those pages.
+  pi = 0;
+  _next_pi = min(_next_pi, _pages.size());
+  while (pi < _next_pi) {
+    VertexDataBlock *block = _pages[pi]->alloc(size);
+    if (block != (VertexDataBlock *)NULL) {
+      _next_pi = pi;
+      return block;
+    }
+    if (_pages[pi]->get_total_size() == 0) {
+      // This page is empty, but must have been too small.  Create a
+      // new page in its place.
+      delete _pages[pi];
+      _pages[pi] = create_new_page(size);
+      return _pages[pi]->alloc(size);
+    }
+    ++pi;
+  }
+
+  // No page was good enough.  Create a new page.  Make it at least
+  // large enough to hold this requested block.
+  VertexDataPage *page = create_new_page(size);
+  _pages.push_back(page);
+  return page->alloc(size);
+}
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::Constructor
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VertexDataPage::
+VertexDataPage(size_t page_size) : SimpleAllocator(page_size), SimpleLruPage(page_size) {
+  _page_data = new unsigned char[get_max_size()];
+  _size = page_size;
+  _uncompressed_size = _size;
+  _saved_block = NULL;
+  _total_page_size += _size;
+  get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+  set_ram_class(RC_resident);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::Destructor
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VertexDataPage::
+~VertexDataPage() {
+  _total_page_size -= _size;
+  get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+
+  if (_page_data != NULL) {
+    delete[] _page_data;
+  }
+
+  if (_saved_block != (SimpleAllocatorBlock *)NULL) {
+    delete _saved_block;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::make_resident
+//       Access: Published
+//  Description: Moves the page to fully resident status by
+//               expanding it or reading it from disk as necessary.
+////////////////////////////////////////////////////////////////////
+void VertexDataPage::
+make_resident() {
+  if (_ram_class == RC_resident) {
+    // If we're already resident, just mark the page recently used.
+    mark_used_lru();
+    return;
+  }
+
+  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
+    restore_from_disk();
+  }
+
+  if (_ram_class == RC_compressed) {
+#ifdef HAVE_ZLIB
+    PStatTimer timer(_vdata_decompress_pcollector);
+
+    if (gobj_cat.is_debug()) {
+      gobj_cat.debug()
+        << "Expanding page from " << _size
+        << " to " << _uncompressed_size << "\n";
+    }
+    unsigned char *new_data = new unsigned char[_uncompressed_size];
+    uLongf dest_len = _uncompressed_size;
+    int result = uncompress(new_data, &dest_len, _page_data, _size);
+    if (result != Z_OK) {
+      gobj_cat.error()
+        << "Couldn't expand: zlib error " << result << "\n";
+      nassert_raise("zlib error");
+    }
+    nassertv(dest_len == _uncompressed_size);
+
+    get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+    _total_page_size -= _size;
+
+    delete[] _page_data;
+    _page_data = new_data;
+    _size = _uncompressed_size;
+
+    get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+    _total_page_size += _size;
+  
+#endif
+    set_lru_size(_size);
+    set_ram_class(RC_resident);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::make_compressed
+//       Access: Published
+//  Description: Moves the page to compressed status by
+//               compressing it or reading it from disk as necessary.
+////////////////////////////////////////////////////////////////////
+void VertexDataPage::
+make_compressed() {
+  if (_ram_class == RC_compressed) {
+    // If we're already compressed, just mark the page recently used.
+    mark_used_lru();
+    return;
+  }
+
+  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
+    restore_from_disk();
+  }
+
+  if (_ram_class == RC_resident) {
+    nassertv(_size == _uncompressed_size);
+
+#ifdef HAVE_ZLIB
+    PStatTimer timer(_vdata_compress_pcollector);
+
+    // According to the zlib manual, we need to provide this much
+    // buffer to the compress algorithm: 0.1% bigger plus twelve
+    // bytes.
+    uLongf buffer_size = _uncompressed_size + ((_uncompressed_size + 999) / 1000) + 12;
+    Bytef *buffer = (Bytef *)alloca(buffer_size);
+
+    int result = compress2(buffer, &buffer_size,
+                           _page_data, _uncompressed_size,
+                           vertex_data_compression_level);
+    if (result != Z_OK) {
+      gobj_cat.error()
+        << "Couldn't compress: zlib error " << result << "\n";
+      nassert_raise("zlib error");
+    }
+    
+    unsigned char *new_data = new unsigned char[buffer_size];
+    memcpy(new_data, buffer, buffer_size);
+
+    get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+    _total_page_size -= _size;
+
+    delete[] _page_data;
+    _page_data = new_data;
+    _size = buffer_size;
+
+    get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+    _total_page_size += _size;
+
+    if (gobj_cat.is_debug()) {
+      gobj_cat.debug()
+        << "Compressed " << *this << " from " << _uncompressed_size
+        << " to " << _size << "\n";
+    }
+#endif
+    set_lru_size(_size);
+    set_ram_class(RC_compressed);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::make_disk
+//       Access: Published
+//  Description: Moves the page to disk status by
+//               writing it to disk as necessary.
+////////////////////////////////////////////////////////////////////
+void VertexDataPage::
+make_disk() {
+  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
+    // If we're already compressed, just mark the page recently used.
+    mark_used_lru();
+    return;
+  }
+
+  if (_ram_class == RC_resident || _ram_class == RC_compressed) {
+    nassertv(_saved_block == (SimpleAllocatorBlock *)NULL);
+    PStatTimer timer(_vdata_save_pcollector);
+
+    if (gobj_cat.is_debug()) {
+      gobj_cat.debug()
+        << "Storing page, " << _size << " bytes, to disk\n";
+    }
+
+    _saved_block = get_save_file()->write_data(_page_data, _size);
+    if (_saved_block == NULL) {
+      // Can't write it to disk.  Too bad.
+      mark_used_lru();
+      return;
+    }
+
+    get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+    _total_page_size -= _size;
+
+    delete[] _page_data;
+    _page_data = NULL;
+    _size = 0;
+
+    if (_ram_class == RC_resident) {
+      set_ram_class(RC_disk);
+    } else {
+      set_ram_class(RC_compressed_disk);
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::restore_from_disk
+//       Access: Published
+//  Description: Restores the page from disk and makes it
+//               either compressed or resident (according to whether
+//               it was stored compressed on disk).
+////////////////////////////////////////////////////////////////////
+void VertexDataPage::
+restore_from_disk() {
+  if (_ram_class == RC_disk || _ram_class == RC_compressed_disk) {
+    nassertv(_saved_block != (VertexDataBlock *)NULL);
+    nassertv(_page_data == (unsigned char *)NULL && _size == 0);
+
+    PStatTimer timer(_vdata_restore_pcollector);
+
+    size_t buffer_size = _saved_block->get_size();
+    if (gobj_cat.is_debug()) {
+      gobj_cat.debug()
+        << "Restoring page, " << buffer_size << " bytes, from disk\n";
+    }
+
+    unsigned char *new_data = new unsigned char[buffer_size];
+    if (!get_save_file()->read_data(new_data, buffer_size, _saved_block)) {
+      nassert_raise("read error");
+    }
+
+    _page_data = new_data;
+    _size = buffer_size;
+
+    get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+    _total_page_size += _size;
+
+    delete _saved_block;
+    _saved_block = NULL;
+
+    set_lru_size(_size);
+    if (_ram_class == RC_compressed_disk) {
+      set_ram_class(RC_compressed);
+    } else {
+      set_ram_class(RC_resident);
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::make_block
+//       Access: Protected, Virtual
+//  Description: Creates a new SimpleAllocatorBlock object.  Override
+//               this function to specialize the block type returned.
+////////////////////////////////////////////////////////////////////
+SimpleAllocatorBlock *VertexDataPage::
+make_block(size_t start, size_t size) {
+  return new VertexDataBlock(this, start, size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::evict_lru
+//       Access: Public, Virtual
+//  Description: Evicts the page from the LRU.  Called internally when
+//               the LRU determines that it is full.  May also be
+//               called externally when necessary to explicitly evict
+//               the page.
+//
+//               It is legal for this method to either evict the page
+//               as requested, do nothing (in which case the eviction
+//               will be requested again at the next epoch), or
+//               requeue itself on the tail of the queue (in which
+//               case the eviction will be requested again much
+//               later).
+////////////////////////////////////////////////////////////////////
+void VertexDataPage::
+evict_lru() {
+  switch (_ram_class) {
+  case RC_resident:
+    if (_compressed_lru.get_max_size() == 0) {
+      make_disk();
+    } else {
+      make_compressed();
+    }
+    break;
+
+  case RC_compressed:
+    make_disk();
+    break;
+
+  case RC_disk:
+  case RC_compressed_disk:
+    gobj_cat.warning()
+      << "Cannot evict array data from disk.\n";
+    break;
+
+  case RC_end_of_list:
+    break;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataPage::make_save_file
+//       Access: Private, Static
+//  Description: Creates the global VertexDataSaveFile that will be
+//               used to save vertex data buffers to disk when
+//               necessary.
+////////////////////////////////////////////////////////////////////
+void VertexDataPage::
+make_save_file() {
+  size_t max_size = (size_t)max_disk_vertex_data;
+
+  _save_file = new VertexDataSaveFile(vertex_save_file_directory,
+                                      vertex_save_file_prefix, max_size);
+}

panda/src/gobj/vertexDataBook.h

@@ -0,0 +1,159 @@
+// Filename: vertexDataBook.h
+// Created by:  drose (16May07)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001 - 2004, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://etc.cmu.edu/panda3d/docs/license/ .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef VERTEXDATAPAGE_H
+#define VERTEXDATAPAGE_H
+
+#include "pandabase.h"
+#include "simpleLru.h"
+#include "simpleAllocator.h"
+#include "referenceCount.h"
+#include "pStatCollector.h"
+
+class VertexDataPage;
+class VertexDataBlock;
+class VertexDataSaveFile;
+
+////////////////////////////////////////////////////////////////////
+//       Class : VertexDataBook
+// Description : A collection of VertexDataPages, which can be used to
+//               allocate new VertexDataBlock objects.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDA VertexDataBook {
+PUBLISHED:
+  VertexDataBook(size_t block_size);
+  ~VertexDataBook();
+
+  VertexDataBlock *alloc(size_t size);
+
+  INLINE int get_num_pages() const;
+  INLINE VertexDataPage *get_page(int n) const;
+
+private:
+  INLINE VertexDataPage *create_new_page(size_t size);
+
+private:
+  size_t _block_size;
+  typedef pvector<VertexDataPage *> Pages;
+  Pages _pages;
+  size_t _next_pi;
+};
+
+////////////////////////////////////////////////////////////////////
+//       Class : VertexDataPage
+// Description : A block of bytes that holds one or more
+//               VertexDataBlocks.  The entire page may be paged out,
+//               in the form of in-memory compression or to an on-disk
+//               cache file, if necessary.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDA VertexDataPage : public SimpleAllocator, public SimpleLruPage {
+PUBLISHED:
+  VertexDataPage(size_t page_size);
+  ~VertexDataPage();
+
+  // These are used to indicate the current residency state of the
+  // page, which may or may not have been temporarily evicted to
+  // satisfy memory requirements.
+  enum RamClass {
+    RC_resident,
+    RC_compressed,
+    RC_disk,
+    RC_compressed_disk,
+
+    RC_end_of_list,  // list marker; do not use
+  };
+
+  INLINE RamClass get_ram_class() const;
+  INLINE void check_resident() const;
+
+  void make_resident();
+  void make_compressed();
+  void make_disk();
+  void restore_from_disk();
+
+  INLINE VertexDataBlock *alloc(size_t size);
+  INLINE VertexDataBlock *get_first_block() const;
+
+  INLINE static size_t get_total_page_size();
+  INLINE static SimpleLru *get_global_lru(RamClass rclass);
+  INLINE static VertexDataSaveFile *get_save_file();
+
+public:
+  INLINE unsigned char *get_page_data() const;
+
+protected:
+  virtual SimpleAllocatorBlock *make_block(size_t start, size_t size);
+  virtual void evict_lru();
+
+private:
+  INLINE void set_ram_class(RamClass ram_class);
+  static void make_save_file();
+
+  unsigned char *_page_data;
+  size_t _size, _uncompressed_size;
+  RamClass _ram_class;
+  SimpleAllocatorBlock *_saved_block;
+
+  static SimpleLru _resident_lru;
+  static SimpleLru _compressed_lru;
+  static SimpleLru _disk_lru;
+  static SimpleLru *_global_lru[RC_end_of_list];
+
+  static size_t _total_page_size;
+  static VertexDataSaveFile *_save_file;
+
+  static PStatCollector _vdata_compress_pcollector;
+  static PStatCollector _vdata_decompress_pcollector;
+  static PStatCollector _vdata_save_pcollector;
+  static PStatCollector _vdata_restore_pcollector;
+
+public:
+  static TypeHandle get_class_type() {
+    return _type_handle;
+  }
+  static void init_type() {
+    register_type(_type_handle, "VertexDataPage");
+  }
+
+private:
+  static TypeHandle _type_handle;
+};
+
+////////////////////////////////////////////////////////////////////
+//       Class : VertexDataBlock
+// Description : A block of bytes that stores the actual raw vertex
+//               data referenced by a GeomVertexArrayData object.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDA VertexDataBlock : public SimpleAllocatorBlock, public ReferenceCount {
+protected:
+  INLINE VertexDataBlock(VertexDataPage *page,
+                         size_t start, size_t size);
+
+PUBLISHED:
+  INLINE VertexDataPage *get_page() const;
+  INLINE VertexDataBlock *get_next_block() const;
+
+public:
+  INLINE unsigned char *get_pointer() const;
+
+  friend class VertexDataPage;
+};
+
+#include "vertexDataBook.I"
+
+#endif

panda/src/gobj/vertexDataBuffer.I

@@ -0,0 +1,161 @@
+// Filename: vertexDataBuffer.I
+// Created by:  drose (14May07)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001 - 2004, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://etc.cmu.edu/panda3d/docs/license/ .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBuffer::
+VertexDataBuffer() :
+  _resident_data(NULL),
+  _size(0)
+{
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBuffer::
+VertexDataBuffer(size_t size) :
+  _resident_data(NULL),
+  _size(0)
+{
+  unclean_realloc(size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::Copy Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBuffer::
+VertexDataBuffer(const VertexDataBuffer &copy) :
+  _resident_data(NULL),
+  _size(0)
+{
+  (*this) = copy;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::Copy Assignment Operator
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataBuffer::
+operator = (const VertexDataBuffer &copy) {
+  unclean_realloc(copy.get_size());
+  memcpy(_resident_data, copy.get_read_pointer(), _size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::Destructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE VertexDataBuffer::
+~VertexDataBuffer() {
+  clear();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::get_read_pointer
+//       Access: Public
+//  Description: Returns a read-only pointer to the raw data.
+////////////////////////////////////////////////////////////////////
+INLINE const unsigned char *VertexDataBuffer::
+get_read_pointer() const {
+  if (_block != (VertexDataBlock *)NULL) {
+    // We don't necessarily need to page the buffer all the way into
+    // independent status; it's sufficient just to return the block's
+    // pointer, which will force its page to resident status.
+    return _block->get_pointer();
+  }
+  return _resident_data;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::get_write_pointer
+//       Access: Public
+//  Description: Returns a writable pointer to the raw data.
+////////////////////////////////////////////////////////////////////
+INLINE unsigned char *VertexDataBuffer::
+get_write_pointer() {
+  if (_block != (VertexDataBlock *)NULL) {
+    page_in();
+  }
+  return _resident_data;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::get_size
+//       Access: Public
+//  Description: Returns the number of bytes in the buffer.
+////////////////////////////////////////////////////////////////////
+INLINE size_t VertexDataBuffer::
+get_size() const {
+  return _size;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::unclean_realloc
+//       Access: Public
+//  Description: Changes the size of the buffer, without regard to
+//               preserving its data.  The buffer may contain random
+//               data after this call.
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataBuffer::
+unclean_realloc(size_t size) {
+  // At the moment, this has no distinct definition, since the system
+  // realloc() call doesn't have an unclean variant.
+  clean_realloc(size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::clear
+//       Access: Public
+//  Description: Empties the buffer and sets its size to 0.
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataBuffer::
+clear() {
+  unclean_realloc(0);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::swap
+//       Access: Public
+//  Description: Swaps the data buffers between this one and the other
+//               one.
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataBuffer::
+swap(VertexDataBuffer &other) {
+  unsigned char *resident_data = _resident_data;
+  size_t size = _size;
+  PT(VertexDataBlock) block = _block;
+
+  _resident_data = other._resident_data;
+  _size = other._size;
+  _block = other._block;
+
+  other._resident_data = resident_data;
+  other._size = size;
+  other._block = block;
+}

panda/src/gobj/vertexDataBuffer.cxx

@@ -0,0 +1,120 @@
+// Filename: vertexDataBuffer.cxx
+// Created by:  drose (14May07)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001 - 2004, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://etc.cmu.edu/panda3d/docs/license/ .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "vertexDataBuffer.h"
+
+TypeHandle VertexDataBuffer::_type_handle;
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::clean_realloc
+//       Access: Public
+//  Description: Changes the size of the buffer, preserving its data
+//               (except for any data beyond the new end of the
+//               buffer, if the buffer is being reduced).  If the
+//               buffer is expanded, the new data is uninitialized.
+////////////////////////////////////////////////////////////////////
+void VertexDataBuffer::
+clean_realloc(size_t size) {
+  if (size != _size) {
+    if (size == 0) {
+      // If we're going to size 0, we don't necessarily need to page
+      // in first.
+      if (_block != (VertexDataBlock *)NULL) {
+        // We're currently paged out.  Discard the page.
+        _block = NULL;
+      } else {
+        // We're currently paged in.  Decrement the global total.
+        get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+      }
+        
+      if (_resident_data != (unsigned char *)NULL) {
+        free(_resident_data);
+        _resident_data = NULL;
+      }
+      _block = NULL;
+      
+    } else {
+      // Page if if we're currently paged out.
+      if (_block != (VertexDataBlock *)NULL) {
+        page_in();
+      }
+      
+      get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+      get_class_type().inc_memory_usage(TypeHandle::MC_array, size);
+    
+      if (_resident_data == (unsigned char *)NULL) {
+        _resident_data = (unsigned char *)malloc(size);
+      } else {
+        _resident_data = (unsigned char *)::realloc(_resident_data, size);
+      }
+      nassertv(_resident_data != (unsigned char *)NULL);
+    }
+
+    _size = size;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::page_out
+//       Access: Public
+//  Description: Moves the buffer out of independent memory and puts
+//               it on a page in the indicated book.  The buffer may
+//               still be directly accessible as long as its page
+//               remains resident.  Any subsequent attempt to rewrite
+//               the buffer will implicitly move it off of the page
+//               and back into independent memory.
+////////////////////////////////////////////////////////////////////
+void VertexDataBuffer::
+page_out(VertexDataBook &book) {
+  if (_block != (VertexDataBlock *)NULL || _size == 0) {
+    // We're already paged out.
+    return;
+  }
+  nassertv(_resident_data != (unsigned char *)NULL);
+
+  _block = book.alloc(_size);
+  memcpy(_block->get_pointer(), _resident_data, _size);
+  free(_resident_data);
+  _resident_data = NULL;
+
+  get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::page_in
+//       Access: Public
+//  Description: Moves the buffer off of its current page and into
+//               independent memory.  If the page is not already
+//               resident, it is forced resident first.
+////////////////////////////////////////////////////////////////////
+void VertexDataBuffer::
+page_in() {
+  if (_block == (VertexDataBlock *)NULL) {
+    // We're already paged in.
+    return;
+  }
+
+  nassertv(_resident_data == (unsigned char *)NULL);
+
+  _resident_data = (unsigned char *)malloc(_size);
+  nassertv(_resident_data != (unsigned char *)NULL);
+  memcpy(_resident_data, _block->get_pointer(), _size);
+  _block = NULL;
+  
+  get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+}

panda/src/gobj/vertexDataBuffer.h

@@ -0,0 +1,71 @@
+// Filename: vertexDataBuffer.h
+// Created by:  drose (14May07)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001 - 2004, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://etc.cmu.edu/panda3d/docs/license/ .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef VERTEXDATABUFFER_H
+#define VERTEXDATABUFFER_H
+
+#include "pandabase.h"
+#include "vertexDataBook.h"
+#include "pointerTo.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : VertexDataBuffer
+// Description : A block of bytes that stores the actual raw vertex
+//               data referenced by a GeomVertexArrayData object.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDA VertexDataBuffer {
+public:
+  INLINE VertexDataBuffer();
+  INLINE VertexDataBuffer(size_t size);
+  INLINE VertexDataBuffer(const VertexDataBuffer &copy);
+  INLINE void operator = (const VertexDataBuffer &copy);
+  INLINE ~VertexDataBuffer();
+
+  INLINE const unsigned char *get_read_pointer() const;
+  INLINE unsigned char *get_write_pointer();
+
+  INLINE size_t get_size() const;
+  void clean_realloc(size_t size);
+  INLINE void unclean_realloc(size_t size);
+  INLINE void clear();
+
+  INLINE void swap(VertexDataBuffer &other);
+
+  void page_out(VertexDataBook &book);
+  void page_in();
+
+private:
+  unsigned char *_resident_data;
+  size_t _size;
+  PT(VertexDataBlock) _block;
+
+public:
+  static TypeHandle get_class_type() {
+    return _type_handle;
+  }
+  static void init_type() {
+    register_type(_type_handle, "VertexDataBuffer");
+  }
+
+private:
+  static TypeHandle _type_handle;
+};
+
+#include "vertexDataBuffer.I"
+
+#endif

panda/src/gobj/vertexDataSaveFile.I

@@ -29,3 +29,25 @@ is_valid() const {
   return _is_valid;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataSaveFile::get_total_file_size
+//       Access: Public
+//  Description: Returns the amount of space consumed by the save
+//               file, including unused portions.
+////////////////////////////////////////////////////////////////////
+INLINE size_t VertexDataSaveFile::
+get_total_file_size() const {
+  return _total_file_size;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataSaveFile::get_used_file_size
+//       Access: Public
+//  Description: Returns the amount of space within the save file that
+//               is currently in use.
+////////////////////////////////////////////////////////////////////
+INLINE size_t VertexDataSaveFile::
+get_used_file_size() const {
+  return _allocator.get_total_size();
+}
+

panda/src/gobj/vertexDataSaveFile.cxx

@@ -36,6 +36,7 @@ VertexDataSaveFile(const Filename &directory, const string &prefix,
   }
 
   _is_valid = false;
+  _total_file_size = 0;
 
   // Try to open and lock a writable temporary filename.
   int index = 0;
@@ -209,6 +210,7 @@ write_data(const unsigned char *data, size_t size) {
     }
 #endif  // _WIN32
 
+    _total_file_size = max(_total_file_size, block->get_start() + size);
   }
 
   return block;

panda/src/gobj/vertexDataSaveFile.h

@@ -42,6 +42,9 @@ public:
   ~VertexDataSaveFile();
 
   INLINE bool is_valid() const;
+
+  INLINE size_t get_total_file_size() const;
+  INLINE size_t get_used_file_size() const;
   
   SimpleAllocatorBlock *write_data(const unsigned char *data, size_t size);
   bool read_data(unsigned char *data, size_t size,
@@ -51,6 +54,7 @@ private:
   SimpleAllocator _allocator;
   Filename _filename;
   bool _is_valid;
+  size_t _total_file_size;
 
 #ifdef _WIN32
   HANDLE _handle;

panda/src/pgraph/geomTransformer.cxx

@@ -400,7 +400,6 @@ apply_state(GeomNode *node, const RenderState *state) {
 int GeomTransformer::
 collect_vertex_data(Geom *geom, int collect_bits) {
   CPT(GeomVertexData) vdata = geom->get_vertex_data();
-
   if (vdata->get_num_rows() > _max_collect_vertices) {
     // Don't even bother.
     return 0;
@@ -587,6 +586,7 @@ collect_vertex_data(Geom *geom, int collect_bits) {
     // The TransformBlendTable.  This one is the easiest, because we
     // can modify it directly, and it will uniquify blend objects for
     // us.
+
     CPT(TransformBlendTable) old_btable;
     if (vdata->get_transform_blend_table() != (TransformBlendTable *)NULL) {
       old_btable = vdata->get_transform_blend_table();
@@ -658,7 +658,6 @@ collect_vertex_data(Geom *geom, int collect_bits) {
     new_data->set_slider_table(SliderTable::register_table(new_sliders));
   }
 
-
   AlreadyCollectedData &acd = _already_collected[vdata];
   acd._data = new_data;
   acd._offset = offset;

panda/src/pstatclient/pStatProperties.cxx

@@ -200,10 +200,13 @@ static LevelCollectorProperties level_properties[] = {
   { 1, "Main memory:C++",                  { 0.2, 0.2, 1.0 } },
   { 1, "Main memory:Interpreter",          { 0.8, 0.2, 0.5 } },
   { 1, "Vertex Data",                      { 1.0, 0.4, 0.0 },  "MB", 64, 1048576 },
+  { 1, "Vertex Data:Small",                { 0.2, 0.3, 0.4 } },
+  { 1, "Vertex Data:Independent",          { 0.9, 0.1, 0.9 } },
   { 1, "Vertex Data:Disk",                 { 0.6, 0.9, 0.1 } },
+  { 1, "Vertex Data:Disk:Unused",          { 0.8, 0.4, 0.5 } },
+  { 1, "Vertex Data:Disk:Used",            { 0.2, 0.1, 0.6 } },
   { 1, "Vertex Data:Compressed",           { 0.5, 0.1, 0.4 } },
-  { 1, "Vertex Data:Resident",             { 0.9, 0.1, 0.7 } },
-  { 1, "Vertex Data:Active",               { 0.5, 0.7, 0.9 } },
+  { 1, "Vertex Data:Resident",             { 0.9, 1.0, 0.7 } },
   { 1, "TransformStates",                  { 1.0, 0.5, 0.5 },  "", 5000 },
   { 1, "TransformStates:On nodes",         { 0.2, 0.8, 1.0 } },
   { 1, "TransformStates:Cached",           { 1.0, 0.0, 0.2 } },

panda/src/putil/linkedListNode.I

@@ -54,6 +54,19 @@ INLINE LinkedListNode::
   nassertv((_next == NULL && _prev == NULL) || (_next == this && _prev == this));
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: LinkedListNode::is_on_list
+//       Access: Protected
+//  Description: Returns true if the node is member of any list, false
+//               if it has been removed or never added.  The head of a
+//               list generally appears to to always be a member of
+//               itself.
+////////////////////////////////////////////////////////////////////
+INLINE bool LinkedListNode::
+is_on_list() const {
+  return (_next != NULL);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: LinkedListNode::remove_from_list
 //       Access: Protected
@@ -62,6 +75,7 @@ INLINE LinkedListNode::
 ////////////////////////////////////////////////////////////////////
 INLINE void LinkedListNode::
 remove_from_list() {
+  nassertv(_prev != NULL && _next != NULL);
   nassertv(_prev->_next == this && _next->_prev == this);
   _prev->_next = _next;
   _next->_prev = _prev;

panda/src/putil/linkedListNode.h

@@ -38,6 +38,7 @@ protected:
   INLINE LinkedListNode(bool);
   INLINE ~LinkedListNode();
 
+  INLINE bool is_on_list() const;
   INLINE void remove_from_list();
   INLINE void insert_before(LinkedListNode *node);
   INLINE void insert_after(LinkedListNode *node);