add model-cache-max-kbytes

panda/src/putil/bamCache.I

@@ -17,6 +17,19 @@
 ////////////////////////////////////////////////////////////////////
 
 
+////////////////////////////////////////////////////////////////////
+//     Function: BamCache::set_active
+//       Access: Published
+//  Description: Changes the state of the active flag.  "active" means
+//               that the cache should be consulted automatically on
+//               loads, "not active" means that objects should be
+//               loaded directly without consulting the cache.
+////////////////////////////////////////////////////////////////////
+INLINE void BamCache::
+set_active(bool active) {
+  _active = active;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: BamCache::get_active
 //       Access: Published
@@ -31,19 +44,6 @@ get_active() const {
   return _active;
 }
 
-////////////////////////////////////////////////////////////////////
-//     Function: BamCache::set_active
-//       Access: Published
-//  Description: Changes the state of the active flag.  "active" means
-//               that the cache should be consulted automatically on
-//               loads, "not active" means that objects should be
-//               loaded directly without consulting the cache.
-////////////////////////////////////////////////////////////////////
-INLINE void BamCache::
-set_active(bool active) {
-  _active = active;
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: BamCache::get_root
 //       Access: Published
@@ -55,6 +55,60 @@ get_root() const {
   return _root;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: BamCache::set_flush_time
+//       Access: Published
+//  Description: Specifies the time in seconds between automatic
+//               flushes of the cache index.
+////////////////////////////////////////////////////////////////////
+INLINE void BamCache::
+set_flush_time(int flush_time) {
+  _flush_time = flush_time;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCache::get_flush_time
+//       Access: Published
+//  Description: Returns the time in seconds between automatic
+//               flushes of the cache index.
+////////////////////////////////////////////////////////////////////
+INLINE int BamCache::
+get_flush_time() const {
+  return _flush_time;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCache::set_cache_max_kbytes
+//       Access: Published
+//  Description: Specifies the maximum size, in kilobytes, which the
+//               cache is allowed to grow to.  If a newly cached file
+//               would exceed this size, an older file is removed from
+//               the cache.
+//
+//               Note that in the case of multiple different processes
+//               simultaneously operating on the same cache directory,
+//               the actual cache size may slightlyexceed this value
+//               from time to time due to latency in checking between
+//               the processes.
+////////////////////////////////////////////////////////////////////
+INLINE void BamCache::
+set_cache_max_kbytes(int max_kbytes) {
+  _max_kbytes = max_kbytes;
+  check_cache_size();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCache::get_cache_max_kbytes
+//       Access: Published
+//  Description: Returns the maximum size, in kilobytes, which the
+//               cache is allowed to grow to.  See
+//               set_cache_max_kbytes().
+////////////////////////////////////////////////////////////////////
+INLINE int BamCache::
+get_cache_max_kbytes() const {
+  return _max_kbytes;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: BamCache::get_global_ptr
 //       Access: Published, Static

panda/src/putil/bamCache.cxx

@@ -25,6 +25,8 @@
 #include "bam.h"
 #include "typeRegistry.h"
 #include "string_utils.h"
+#include "configVariableInt.h"
+#include "configVariableString.h"
 
 BamCache *BamCache::_global_ptr = NULL;
 
@@ -39,6 +41,28 @@ BamCache() :
   _index(new BamCacheIndex),
   _index_stale_since(0)
 {
+  ConfigVariableFilename model_cache_dir
+    ("model-cache-dir", Filename(), 
+     PRC_DESC("The full path to a directory, local to this computer, in which "
+              "model and texture files will be cached on load.  If a directory "
+              "name is specified here, files may be loaded from the cache "
+              "instead of from their actual pathnames, which may save load time, "
+              "especially if you are loading egg files instead of bam files, "
+              "or if you are loading models from a shared network drive.  "
+              "If this is the empty string, no cache will be used."));
+  
+  ConfigVariableInt model_cache_flush
+    ("model-cache-flush", 30,
+     PRC_DESC("This is the amount of time, in seconds, between automatic "
+              "flushes of the model-cache index."));
+  
+  ConfigVariableInt model_cache_max_kbytes
+    ("model-cache-max-kbytes", 1048576,
+     PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
+
+  _flush_time = model_cache_flush;
+  _max_kbytes = model_cache_max_kbytes;
+  set_root(model_cache_dir);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -85,6 +109,7 @@ set_root(const Filename &root) {
   _index = new BamCacheIndex;
   _index_stale_since = 0;
   read_index();
+  check_cache_size();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -242,7 +267,7 @@ void BamCache::
 consider_flush_index() {
   if (_index_stale_since != 0) {
     int elapsed = (int)time(NULL) - (int)_index_stale_since;
-    if (elapsed > model_cache_flush) {
+    if (elapsed > _flush_time) {
       flush_index();
     }
   }
@@ -292,6 +317,7 @@ flush_index() {
     _index_ref_contents = orig_index;
     read_index();
   }
+  check_cache_size();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -380,6 +406,8 @@ merge_index(BamCacheIndex *new_index) {
   }
 
   BamCacheIndex *old_index = _index;
+  old_index->release_records();
+  new_index->release_records();
   _index = new BamCacheIndex;
 
   BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
@@ -394,7 +422,6 @@ merge_index(BamCacheIndex *new_index) {
       Filename cache_pathname(_root, record->get_cache_filename());
       if (cache_pathname.exists()) {
         // The file exists; keep it.
-        _index->_cache_size += record->_record_size;
         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
       }
       ++ai;
@@ -405,7 +432,6 @@ merge_index(BamCacheIndex *new_index) {
       Filename cache_pathname(_root, record->get_cache_filename());
       if (cache_pathname.exists()) {
         // The file exists; keep it.
-        _index->_cache_size += record->_record_size;
         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
       }
       ++bi;
@@ -417,7 +443,6 @@ merge_index(BamCacheIndex *new_index) {
       if (*a_record == *b_record) {
         // They're the same entry.  It doesn't really matter which one
         // we keep.
-        _index->_cache_size += a_record->_record_size;
         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
 
       } else {
@@ -429,7 +454,6 @@ merge_index(BamCacheIndex *new_index) {
         if (cache_pathname.exists()) {
           PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
           if (record != (BamCacheRecord *)NULL) {
-            _index->_cache_size += record->_record_size;
             _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
           }
         }
@@ -447,7 +471,6 @@ merge_index(BamCacheIndex *new_index) {
     Filename cache_pathname(_root, record->get_cache_filename());
     if (cache_pathname.exists()) {
       // The file exists; keep it.
-      _index->_cache_size += record->_record_size;
       _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
     }
     ++ai;
@@ -459,11 +482,12 @@ merge_index(BamCacheIndex *new_index) {
     Filename cache_pathname(_root, record->get_cache_filename());
     if (cache_pathname.exists()) {
       // The file exists; keep it.
-      _index->_cache_size += record->_record_size;
       _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
     }
     ++bi;
   }
+
+  _index->process_new_records();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -505,13 +529,14 @@ rebuild_index() {
           util_cat.info()
             << "Multiple cache files defining " << record->get_source_pathname() << "\n";
           pathname.unlink();
-        } else {
-          _index->_cache_size += record->_record_size;
         }
       }
     }
   }
+  _index->process_new_records();
+
   _index_stale_since = time(NULL);
+  check_cache_size();
   flush_index();
 }
 
@@ -519,27 +544,16 @@ rebuild_index() {
 //     Function: BamCache::add_to_index
 //       Access: Private
 //  Description: Updates the index entry for the indicated record.
+//               Note that a copy of the record is made first.
 ////////////////////////////////////////////////////////////////////
 void BamCache::
 add_to_index(const BamCacheRecord *record) {
   PT(BamCacheRecord) new_record = record->make_copy();
 
-  pair<BamCacheIndex::Records::iterator, bool> result = 
-    _index->_records.insert(BamCacheIndex::Records::value_type(new_record->get_source_pathname(), new_record));
-  if (!result.second) {
-    // We already had a record for this filename; it gets replaced.
-    PT(BamCacheRecord) orig_record = (*result.first).second;
-    if (*orig_record == *record) {
-      // Well, never mind.  The record hasn't changed.
-      return;
-    }
-
-    _index->_cache_size -= orig_record->_record_size;
-    (*result.first).second = new_record;
+  if (_index->add_record(new_record)) {
+    mark_index_stale();
+    check_cache_size();
   }
-
-  _index->_cache_size += new_record->_record_size;
-  mark_index_stale();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -550,16 +564,33 @@ add_to_index(const BamCacheRecord *record) {
 ////////////////////////////////////////////////////////////////////
 void BamCache::
 remove_from_index(const Filename &source_pathname) {
-  BamCacheIndex::Records::iterator ri = _index->_records.find(source_pathname);
-  if (ri == _index->_records.end()) {
-    // No entry for this record; no problem.
+  if (_index->remove_record(source_pathname)) {
+    mark_index_stale();
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCache::check_cache_size
+//       Access: Private
+//  Description: If the cache size has exceeded its specified size
+//               limit, removes an old file.
+////////////////////////////////////////////////////////////////////
+void BamCache::
+check_cache_size() {
+  if (_index->_cache_size == 0) {
+    // 0 means no limit.
     return;
   }
 
-  PT(BamCacheRecord) record = (*ri).second;
-  _index->_cache_size -= record->_record_size;
-  _index->_records.erase(ri);
-  mark_index_stale();
+  if (_index->_cache_size > _max_kbytes * 1024) {
+    while (_index->_cache_size > _max_kbytes * 1024) {
+      PT(BamCacheRecord) record = _index->evict_old_file();
+      nassertv(record != (BamCacheRecord *)NULL);
+      Filename cache_pathname(_root, record->get_cache_filename());
+      cache_pathname.unlink();
+    }
+    mark_index_stale();
+  }
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -894,20 +925,7 @@ void BamCache::
 make_global() {
   _global_ptr = new BamCache;
 
-  ConfigVariableFilename model_cache_dir
-    ("model-cache-dir", Filename(), 
-     PRC_DESC("The full path to a directory, local to this computer, in which "
-              "model and texture files will be cached on load.  If a directory "
-              "name is specified here, files may be loaded from the cache "
-              "instead of from their actual pathnames, which may save load time, "
-              "especially if you are loading egg files instead of bam files, "
-              "or if you are loading models from a shared network drive.  "
-              "If this is the empty string, no cache will be used."));
-  if (model_cache_dir.empty()) {
+  if (_global_ptr->get_root().empty()) {
     _global_ptr->set_active(false);
-
-  } else {
-    _global_ptr->set_active(true);
-    _global_ptr->set_root(model_cache_dir);
   }
 }

panda/src/putil/bamCache.h

@@ -49,11 +49,17 @@ PUBLISHED:
   BamCache();
   ~BamCache();
 
-  INLINE bool get_active() const;
   INLINE void set_active(bool flag);
+  INLINE bool get_active() const;
 
-  INLINE const Filename &get_root() const;
   void set_root(const Filename &root);
+  INLINE const Filename &get_root() const;
+
+  INLINE void set_flush_time(int flush_time);
+  INLINE int get_flush_time() const;
+
+  INLINE void set_cache_max_kbytes(int max_kbytes);
+  INLINE int get_cache_max_kbytes() const;
 
   PT(BamCacheRecord) lookup(const Filename &source_filename, 
                             const string &cache_extension);
@@ -75,6 +81,8 @@ private:
   void add_to_index(const BamCacheRecord *record);
   void remove_from_index(const Filename &source_filename);
 
+  void check_cache_size();
+
   static BamCacheIndex *do_read_index(Filename &index_pathname);
   static bool do_write_index(Filename &index_pathname, const BamCacheIndex *index);
 
@@ -90,6 +98,8 @@ private:
 
   bool _active;
   Filename _root;
+  int _flush_time;
+  int _max_kbytes;
   static BamCache *_global_ptr;
 
   BamCacheIndex *_index;

panda/src/putil/bamCacheIndex.I

@@ -23,7 +23,8 @@
 //  Description:
 ////////////////////////////////////////////////////////////////////
 INLINE BamCacheIndex::
-BamCacheIndex() :
+BamCacheIndex() : 
+  LinkedListNode(true),
   _cache_size(0)
 {
 }

panda/src/putil/bamCacheIndex.cxx

@@ -18,9 +18,25 @@
 
 #include "bamCacheIndex.h"
 #include "indent.h"
+#include <algorithm>
 
 TypeHandle BamCacheIndex::_type_handle;
 
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheIndex::Destructor
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+BamCacheIndex::
+~BamCacheIndex() {
+#ifndef NDEBUG
+  // We need to "empty" the linked list to make the LinkedListNode
+  // destructors happy.
+  release_records();
+#endif
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: BamCacheIndex::write
 //       Access: Public
@@ -33,7 +49,7 @@ write(ostream &out, int indent_level) const {
 
   Records::const_iterator ri;
   for (ri = _records.begin(); ri != _records.end(); ++ri) {
-    PT(BamCacheRecord) record = (*ri).second;
+    BamCacheRecord *record = (*ri).second;
     indent(out, indent_level + 2)
       << setw(10) << record->_record_size << " "
       << record->get_cache_filename() << " "
@@ -44,6 +60,133 @@ write(ostream &out, int indent_level) const {
     << setw(12) << _cache_size << " bytes total\n";
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheIndex::process_new_records
+//       Access: Private
+//  Description: Should be called after the _records index has been
+//               filled externally, this will sort the records by
+//               access time and calculate _cache_size.
+////////////////////////////////////////////////////////////////////
+void BamCacheIndex::
+process_new_records() {
+  nassertv(_cache_size == 0);
+
+  // Fill up a vector so we can sort the records into order by access
+  // time.
+  RecordVector rv;
+  rv.reserve(_records.size());
+
+  Records::const_iterator ri;
+  for (ri = _records.begin(); ri != _records.end(); ++ri) {
+    BamCacheRecord *record = (*ri).second;
+    _cache_size += record->_record_size;
+    rv.push_back(record);
+  }
+
+  sort(rv.begin(), rv.end(), BamCacheRecord::SortByAccessTime());
+
+  // Now put them into the linked list.
+  RecordVector::const_iterator rvi;
+  for (rvi = rv.begin(); rvi != rv.end(); ++rvi) {
+    BamCacheRecord *record = *rvi;
+    record->insert_before(this);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheIndex::release_records
+//       Access: Private
+//  Description: This is the inverse of process_new_records: it
+//               releases the records from the linked list, so that
+//               they may be added to another index or whatever.
+//               Calling this, of course, invalidates the index until
+//               process_new_records() is called again.
+////////////////////////////////////////////////////////////////////
+void BamCacheIndex::
+release_records() {
+  Records::const_iterator ri;
+  for (ri = _records.begin(); ri != _records.end(); ++ri) {
+    BamCacheRecord *record = (*ri).second;
+    record->_next = NULL;
+    record->_prev = NULL;
+  }
+  _next = this;
+  _prev = this;
+  _cache_size = 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheIndex::evict_old_file
+//       Access: Private
+//  Description: Evicts an old file from the cache.  Records the record.
+////////////////////////////////////////////////////////////////////
+PT(BamCacheRecord) BamCacheIndex::
+evict_old_file() {
+  // The first record in the linked list is the least-recently-used
+  // one.
+  nassertr(_next != this, NULL);
+  PT(BamCacheRecord) record = (BamCacheRecord *)_next;
+  bool removed = remove_record(record->get_source_pathname());
+  nassertr(removed, NULL);
+
+  return record;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheIndex::add_record
+//       Access: Private
+//  Description: Adds a newly-created BamCacheRecord into the index.
+//               If a matching record is already in the index, it is
+//               replaced with the new record.  Returns true if the
+//               record was added, or false if the equivalent record
+//               was already there and the index is unchanged.
+////////////////////////////////////////////////////////////////////
+bool BamCacheIndex::
+add_record(BamCacheRecord *record) {
+  pair<Records::iterator, bool> result = 
+    _records.insert(Records::value_type(record->get_source_pathname(), record));
+  if (!result.second) {
+    // We already had a record for this filename; it gets replaced.
+    BamCacheRecord *orig_record = (*result.first).second;
+    orig_record->remove_from_list();
+    if (*orig_record == *record) {
+      // Well, never mind.  The record hasn't changed.
+      orig_record->insert_before(this);
+      return false;
+    }
+
+    _cache_size -= orig_record->_record_size;
+    (*result.first).second = record;
+  }
+  record->insert_before(this);
+
+  _cache_size += record->_record_size;
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheIndex::remove_record
+//       Access: Private
+//  Description: Searches for the matching record in the index and
+//               removes it if it is found.  Returns true if the
+//               record was found and removed, or false if there was
+//               no such record and the index is unchanged.
+////////////////////////////////////////////////////////////////////
+bool BamCacheIndex::
+remove_record(const Filename &source_pathname) {
+  Records::iterator ri = _records.find(source_pathname);
+  if (ri == _records.end()) {
+    // No entry for this record; no problem.
+    return false;
+  }
+
+  BamCacheRecord *record = (*ri).second;
+  record->remove_from_list();
+  _cache_size -= record->_record_size;
+  _records.erase(ri);
+  return true;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: BamCacheIndex::register_with_read_factory
 //       Access: Public, Static
@@ -113,13 +256,13 @@ complete_pointers(TypedWritable **p_list, BamReader *manager) {
       util_cat.info()
         << "Multiple cache files defining " << record->get_source_pathname()
         << " in index.\n";
-    } else {
-      _cache_size += record->_record_size;
     }
   }
 
   _record_vector.clear();
 
+  process_new_records();
+
   return pi;
 }
 

panda/src/putil/bamCacheIndex.h

@@ -24,6 +24,7 @@
 #include "pointerTo.h"
 #include "filename.h"
 #include "typedWritable.h"
+#include "linkedListNode.h"
 #include "pmap.h"
 #include "pvector.h"
 
@@ -37,13 +38,22 @@
 //               For the most part, this class is used only by the
 //               BamCache class.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDA BamCacheIndex : public TypedWritable {
+class EXPCL_PANDA BamCacheIndex : public TypedWritable, public LinkedListNode {
 private:
   INLINE BamCacheIndex();
+  ~BamCacheIndex();
 
 public:
   void write(ostream &out, int indent_level = 0) const;
 
+private:
+  void process_new_records();
+  void release_records();
+  PT(BamCacheRecord) evict_old_file();
+
+  bool add_record(BamCacheRecord *record);
+  bool remove_record(const Filename &source_pathname);
+
 private:
   typedef pmap<Filename, PT(BamCacheRecord) > Records;
 

panda/src/putil/bamCacheRecord.I

@@ -180,3 +180,13 @@ set_data(TypedWritable *data, bool owns_pointer) {
   _data = data;
   _owns_pointer = owns_pointer;
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheRecord::SortByAccessTime::operator ()
+//       Access: Public
+//  Description: Returns true if a sorts before b, false otherwise.
+////////////////////////////////////////////////////////////////////
+INLINE bool BamCacheRecord::SortByAccessTime::
+operator () (const BamCacheRecord *a, const BamCacheRecord *b) const {
+  return (a->_record_access_time < b->_record_access_time);
+}

panda/src/putil/bamCacheRecord.h

@@ -21,6 +21,7 @@
 
 #include "pandabase.h"
 #include "typedWritableReferenceCount.h"
+#include "linkedListNode.h"
 
 class BamWriter;
 class BamReader;
@@ -36,7 +37,8 @@ class FactoryParams;
 //               contains information needed to test the validity of
 //               the cache.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDA BamCacheRecord : public TypedWritableReferenceCount {
+class EXPCL_PANDA BamCacheRecord : public TypedWritableReferenceCount,
+                                   public LinkedListNode {
 private:
   BamCacheRecord();
   BamCacheRecord(const Filename &source_pathname, 
@@ -71,6 +73,12 @@ PUBLISHED:
   void write(ostream &out, int indent_level = 0) const;
 
 private:
+  // This class is used to sort BamCacheRecords by access time.
+  class SortByAccessTime {
+  public:
+    INLINE bool operator () (const BamCacheRecord *a, const BamCacheRecord *b) const;
+  };
+
   static string format_timestamp(time_t timestamp);
 
   Filename _source_pathname;
@@ -126,6 +134,7 @@ private:
 
   friend class BamCache;
   friend class BamCacheIndex;
+  friend class BamCacheRecord::SortByAccessTime;
 };
 
 INLINE ostream &operator << (ostream &out, const BamCacheRecord &record) {

panda/src/putil/config_util.cxx

@@ -158,9 +158,3 @@ ConfigVariableDouble sleep_precision
 
 ConfigVariableDouble average_frame_rate_interval
 ("average-frame-rate-interval", 1.0);
-
-ConfigVariableInt model_cache_flush
-("model-cache-flush", 30,
- PRC_DESC("This is the amount of time, in seconds, between automatic "
-          "flushes of the model-cache index, if model-cache-dir is "
-          "in effect."));

panda/src/putil/config_util.h

@@ -24,7 +24,6 @@
 #include "configVariableSearchPath.h"
 #include "configVariableEnum.h"
 #include "configVariableDouble.h"
-#include "configVariableInt.h"
 #include "bamEndian.h"
 #include "bamTextureMode.h"
 #include "dconfig.h"
@@ -62,6 +61,4 @@ extern ConfigVariableDouble max_dt;
 extern ConfigVariableDouble sleep_precision;
 extern ConfigVariableDouble average_frame_rate_interval;
 
-extern ConfigVariableInt model_cache_flush;
-
 #endif /* __CONFIG_UTIL_H__ */