work-in-progress: cache-check-timestamps

panda/src/egg/eggData.I

@@ -23,6 +23,7 @@ EggData() {
   _auto_resolve_externals = false;
   _had_absolute_pathnames = false;
   _coordsys = CS_default;
+  _egg_timestamp = 0;
 }
 
 
@@ -37,7 +38,8 @@ EggData(const EggData &copy) :
   _auto_resolve_externals(copy._auto_resolve_externals),
   _had_absolute_pathnames(copy._had_absolute_pathnames),
   _coordsys(copy._coordsys),
-  _egg_filename(copy._egg_filename) 
+  _egg_filename(copy._egg_filename),
+  _egg_timestamp(copy._egg_timestamp)
 {
 }
 
@@ -53,6 +55,7 @@ operator = (const EggData &copy) {
   _had_absolute_pathnames = copy._had_absolute_pathnames;
   _coordsys = copy._coordsys;
   _egg_filename = copy._egg_filename;
+  _egg_timestamp = copy._egg_timestamp;
   return *this;
 }
 
@@ -133,6 +136,30 @@ get_egg_filename() const {
   return _egg_filename;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: EggData::set_egg_timestamp
+//       Access: Public
+//  Description: Sets the timestamp of the egg file on disk, at the
+//               time it was opened for reading.  This is also
+//               implicitly set by read().
+////////////////////////////////////////////////////////////////////
+INLINE void EggData::
+set_egg_timestamp(time_t egg_timestamp) {
+  _egg_timestamp = egg_timestamp;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: EggData::get_egg_timestamp
+//       Access: Public
+//  Description: Returns the timestamp of the egg file on disk, at the
+//               time it was opened for reading, or 0 if this
+//               information is not available.
+////////////////////////////////////////////////////////////////////
+INLINE time_t EggData::
+get_egg_timestamp() const {
+  return _egg_timestamp;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: EggData::recompute_vertex_normals
 //       Access: Public

panda/src/egg/eggData.cxx

@@ -77,8 +77,15 @@ read(Filename filename, string display_name) {
   }
 
   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-    
-  istream *file = vfs->open_read_file(filename, true);
+
+  PT(VirtualFile) vfile = vfs->get_file(filename);
+  if (vfile == NULL) {
+    egg_cat.error() << "Could not find " << display_name << "\n";
+    return false;
+  }
+  set_egg_timestamp(vfile->get_timestamp());
+
+  istream *file = vfile->open_read_file(true);
   if (file == (istream *)NULL) {
     egg_cat.error() << "Unable to open " << display_name << "\n";
     return false;
@@ -88,7 +95,7 @@ read(Filename filename, string display_name) {
     << "Reading " << display_name << "\n";
   
   bool read_ok = read(*file);
-  vfs->close_read_file(file);
+  vfile->close_read_file(file);
   return read_ok;
 }
 

panda/src/egg/eggData.h

@@ -69,6 +69,9 @@ PUBLISHED:
   INLINE void set_egg_filename(const Filename &egg_filename);
   INLINE const Filename &get_egg_filename() const;
 
+  INLINE void set_egg_timestamp(time_t egg_timestamp);
+  INLINE time_t get_egg_timestamp() const;
+
   INLINE void recompute_vertex_normals(double threshold);
   INLINE void recompute_polygon_normals();
   INLINE void strip_normals();
@@ -84,6 +87,7 @@ private:
   bool _had_absolute_pathnames;
   CoordinateSystem _coordsys;
   Filename _egg_filename;
+  time_t _egg_timestamp;
 
 public:
   static TypeHandle get_class_type() {

panda/src/egg2pg/eggLoader.cxx

@@ -209,7 +209,7 @@ build_graph() {
   //  ((EggGroupNode *)_data)->write(cerr, 0);
 
   // Now build up the scene graph.
-  _root = new ModelRoot(_data->get_egg_filename().get_basename());
+  _root = new ModelRoot(_data->get_egg_filename(), _data->get_egg_timestamp());
   make_node(_data, _root);
 
   reparent_decals();

panda/src/egg2pg/load_egg_file.cxx

@@ -90,9 +90,18 @@ load_egg_file(const Filename &filename, CoordinateSystem cs,
   loader._data->set_coordinate_system(cs);
   loader._record = record;
 
+  PT(VirtualFile) vfile = vfs->get_file(egg_filename);
+  if (vfile == NULL) {
+    return NULL;
+  }
+
+  loader._data->set_egg_timestamp(vfile->get_timestamp());
+
   bool okflag;
-  istream *istr = vfs->open_read_file(egg_filename, true);
+  istream *istr = vfile->open_read_file(true);
   if (istr == (istream *)NULL) {
+    egg2pg_cat.error()
+      << "Couldn't read " << egg_filename << "\n";
     return NULL;
   }
 
@@ -100,7 +109,7 @@ load_egg_file(const Filename &filename, CoordinateSystem cs,
     << "Reading " << egg_filename << "\n";
 
   okflag = loader._data->read(*istr);
-  vfs->close_read_file(istr);
+  vfile->close_read_file(istr);
 
   if (!okflag) {
     egg2pg_cat.error()

panda/src/express/datagramGenerator.cxx

@@ -63,6 +63,18 @@ get_filename() {
   return empty_filename;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: DatagramGenerator::get_timestamp
+//       Access: Published, Virtual
+//  Description: Returns the on-disk timestamp of the file that was
+//               read, at the time it was opened, if that is
+//               available, or 0 if it is not.
+////////////////////////////////////////////////////////////////////
+time_t DatagramGenerator::
+get_timestamp() const {
+  return 0;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: DatagramGenerator::get_file
 //       Access: Published, Virtual

panda/src/express/datagramGenerator.h

@@ -41,6 +41,7 @@ PUBLISHED:
   virtual bool is_error() = 0;
 
   virtual const Filename &get_filename();
+  virtual time_t get_timestamp() const;
   virtual const FileReference *get_file();
   virtual VirtualFile *get_vfile();
   virtual streampos get_file_pos();

panda/src/pgraph/bamFile.cxx

@@ -60,9 +60,6 @@ open_read(const Filename &bam_filename, bool report_errors) {
     return false;
   }
 
-  loader_cat.info()
-    << "Reading " << bam_filename << "\n";
-
   return continue_open_read(bam_filename, report_errors);
 }
 
@@ -220,8 +217,6 @@ bool BamFile::
 open_write(const Filename &bam_filename, bool report_errors) {
   close();
 
-  loader_cat.info() << "Writing " << bam_filename << "\n";
-
   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
   vfs->delete_file(bam_filename);
   if (!_dout.open(bam_filename)) {
@@ -422,6 +417,11 @@ bool BamFile::
 continue_open_read(const string &bam_filename, bool report_errors) {
   _bam_filename = bam_filename;
 
+  if (!_bam_filename.empty()) {
+    loader_cat.info()
+      << "Reading " << _bam_filename << "\n";
+  }
+
   string head;
   if (!_din.read_header(head, _bam_header.size())) {
     if (report_errors) {
@@ -457,6 +457,10 @@ bool BamFile::
 continue_open_write(const string &bam_filename, bool report_errors) {
   _bam_filename = bam_filename;
 
+  if (!_bam_filename.empty()) {
+    loader_cat.info() << "Writing " << _bam_filename << "\n";
+  }
+
   if (!_dout.write_header(_bam_header)) {
     if (report_errors) {
       loader_cat.error() << "Unable to write to " << _bam_filename << "\n";

panda/src/pgraph/loader.cxx

@@ -295,6 +295,12 @@ try_load_file(const Filename &pathname, const LoaderOptions &options,
             << "Model " << pathname << " found in disk cache.\n";
         }
         PT(PandaNode) result = DCAST(PandaNode, record->get_data());
+        if (result->is_of_type(ModelRoot::get_class_type())) {
+          ModelRoot *model_root = DCAST(ModelRoot, result.p());
+          model_root->set_fullpath(pathname);
+          model_root->set_timestamp(record->get_source_timestamp());
+        }
+
         if (premunge_data) {
           SceneGraphReducer sgr;
           sgr.premunge(result, RenderState::make_empty());

panda/src/pgraph/loaderFileTypeBam.cxx

@@ -76,11 +76,21 @@ load_file(const Filename &path, const LoaderOptions &options,
   }
 
   bool report_errors = (options.get_flags() & LoaderOptions::LF_report_errors) != 0;
+
   BamFile bam_file;
   if (!bam_file.open_read(path, report_errors)) {
     return NULL;
   }
   bam_file.get_reader()->set_loader_options(options);
-  return bam_file.read_node(report_errors);
+  time_t timestamp = bam_file.get_reader()->get_source()->get_timestamp();
+
+  PT(PandaNode) node = bam_file.read_node(report_errors);
+  if (node->is_of_type(ModelRoot::get_class_type())) {
+    ModelRoot *model_root = DCAST(ModelRoot, node.p());
+    model_root->set_fullpath(path);
+    model_root->set_timestamp(timestamp);
+  }
+
+  return node;
 }
 

panda/src/pgraph/modelRoot.I

@@ -22,6 +22,21 @@ INLINE ModelRoot::
 ModelRoot(const string &name) :
   ModelNode(name),
   _fullpath(name),
+  _timestamp(0),
+  _reference(new ModelRoot::ModelReference)
+{
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ModelRoot::Constructor
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE ModelRoot::
+ModelRoot(const Filename &fullpath, time_t timestamp) :
+  ModelNode(fullpath.get_basename()),
+  _fullpath(fullpath),
+  _timestamp(timestamp),
   _reference(new ModelRoot::ModelReference)
 {
 }
@@ -73,6 +88,34 @@ set_fullpath(const Filename &fullpath) {
   _fullpath = fullpath;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: ModelRoot::get_timestamp
+//       Access: Published
+//  Description: Returns the timestamp of the file on disk that was
+//               read for this model, at the time it was read, if it
+//               is known.  Returns 0 if the timestamp is not known or
+//               could not be provided.  This can be used as a quick
+//               (but fallible) check to verify whether the file might
+//               have changed since the model was read.
+////////////////////////////////////////////////////////////////////
+INLINE time_t ModelRoot::
+get_timestamp() const {
+  return _timestamp;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ModelRoot::set_timestamp
+//       Access: Published
+//  Description: Sets the timestamp of the file on disk that was read
+//               for this model.  This is normally set automatically
+//               when a model is loaded, and should not be set
+//               directly by the user.
+////////////////////////////////////////////////////////////////////
+INLINE void ModelRoot::
+set_timestamp(time_t timestamp) {
+  _timestamp = timestamp;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: ModelRoot::get_reference
 //       Access: Published
@@ -110,6 +153,7 @@ INLINE ModelRoot::
 ModelRoot(const ModelRoot &copy) :
   ModelNode(copy),
   _fullpath(copy._fullpath),
+  _timestamp(copy._timestamp),
   _reference(copy._reference)
 {
 }

panda/src/pgraph/modelRoot.h

@@ -31,12 +31,16 @@
 class EXPCL_PANDA_PGRAPH ModelRoot : public ModelNode {
 PUBLISHED:
   INLINE ModelRoot(const string &name);
+  INLINE ModelRoot(const Filename &fulllpath, time_t timestamp);
 
   INLINE int get_model_ref_count() const;
 
   INLINE const Filename &get_fullpath() const;
   INLINE void set_fullpath(const Filename &fullpath);
 
+  INLINE time_t get_timestamp() const;
+  INLINE void set_timestamp(time_t timestamp);
+
   // This class is used to unify references to the same model.
   class ModelReference : public ReferenceCount {
   PUBLISHED:
@@ -54,6 +58,7 @@ public:
 
 private:
   Filename _fullpath;
+  time_t _timestamp;
   PT(ModelReference) _reference;
 
 public:

panda/src/putil/bamCache.cxx

@@ -917,10 +917,10 @@ do_read_record(const Filename &cache_pathname, bool read_data) {
   }
   
   // Also get the total file size.
+  PT(VirtualFile) vfile = din.get_vfile();
   istream &in = din.get_stream();
   in.clear();
-  in.seekg(0, ios::end);
-  record->_record_size = in.tellg();
+  record->_record_size = vfile->get_file_size(&in);
 
   // And the last access time is now, duh.
   record->_record_access_time = time(NULL);

panda/src/putil/bamCacheRecord.I

@@ -69,6 +69,19 @@ get_cache_filename() const {
   return _cache_filename;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: BamCacheRecord::get_source_timestamp
+//       Access: Published
+//  Description: Returns the file timestamp of the original source
+//               file that generated this cache record, if available.
+//               In some cases the original file timestamp is not
+//               available, and this will return 0.
+////////////////////////////////////////////////////////////////////
+INLINE time_t BamCacheRecord::
+get_source_timestamp() const {
+  return _source_timestamp;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: BamCacheRecord::get_recorded_time
 //       Access: Published

panda/src/putil/bamCacheRecord.cxx

@@ -28,6 +28,7 @@ BamCacheRecord::
 BamCacheRecord() :
   _recorded_time(0),
   _record_size(0),
+  _source_timestamp(0),
   _ptr(NULL),
   _ref_ptr(NULL),
   _record_access_time(0)
@@ -46,6 +47,7 @@ BamCacheRecord(const Filename &source_pathname,
   _cache_filename(cache_filename),
   _recorded_time(0),
   _record_size(0),
+  _source_timestamp(0),
   _ptr(NULL),
   _ref_ptr(NULL),
   _record_access_time(0)
@@ -64,6 +66,7 @@ BamCacheRecord(const BamCacheRecord &copy) :
   _cache_filename(copy._cache_filename),
   _recorded_time(copy._recorded_time),
   _record_size(copy._record_size),
+  _source_timestamp(copy._source_timestamp),
   _ptr(NULL),
   _ref_ptr(NULL),
   _record_access_time(copy._record_access_time)
@@ -152,6 +155,10 @@ add_dependent_file(const Filename &pathname) {
   } else {
     dfile._timestamp = file->get_timestamp();
     dfile._size = file->get_file_size();
+
+    if (dfile._pathname == _source_pathname) {
+      _source_timestamp = dfile._timestamp;
+    }
   }
 }
 
@@ -174,6 +181,8 @@ void BamCacheRecord::
 write(ostream &out, int indent_level) const {
   indent(out, indent_level)
     << "BamCacheRecord " << get_source_pathname() << "\n";
+  indent(out, indent_level)
+    << "source " << format_timestamp(_source_timestamp) << "\n";
   indent(out, indent_level)
     << "recorded " << format_timestamp(_recorded_time) << "\n";
 
@@ -299,5 +308,11 @@ fillin(DatagramIterator &scan, BamReader *manager) {
     file._pathname = scan.get_string();
     file._timestamp = scan.get_uint32();
     file._size = scan.get_uint64();
+
+    // If we come across the original source file (we normally expect
+    // to), record that as its timestamp.
+    if (file._pathname == _source_pathname) {
+      _source_timestamp = file._timestamp;
+    }
   }
 }

panda/src/putil/bamCacheRecord.h

@@ -52,6 +52,7 @@ PUBLISHED:
 
   INLINE const Filename &get_source_pathname() const;
   INLINE const Filename &get_cache_filename() const;
+  INLINE time_t get_source_timestamp() const;
   INLINE time_t get_recorded_time() const;
 
   INLINE int get_num_dependent_files() const;
@@ -84,6 +85,7 @@ private:
   Filename _cache_filename;
   time_t _recorded_time;
   off_t _record_size;  // this is accurate only in the index file.
+  time_t _source_timestamp;  // Not record to the cache file.
 
   class DependentFile {
   public:

panda/src/putil/config_util.cxx

@@ -146,6 +146,20 @@ ConfigVariableBool preload_simple_textures
           "in a sub-thread.  It's not generally necessary if you are "
           "loading bam files that were generated via egg2bam."));
 
+ConfigVariableBool cache_check_timestamps
+("cache-check-timestamps", true,
+ PRC_DESC("Set this true to check the timestamps on disk (when possible) "
+          "before reloading a file from the in-memory cache, e.g. via ModelPool, "
+          "TexturePool, etc.  When this is false, a model or texture "
+          "that was previously loaded and is still found in the ModelPool is "
+          "immediately returned without consulting the disk, even if the "
+          "file on disk has recently changed.  When this is true, the file "
+          "on disk is always checked to ensure its timestamp has not "
+          "recently changed; and if it has, the in-memory cache is automatically "
+          "invalidated and the file is reloaded from disk.  This is not related "
+          "to on-disk caching via model-cache-dir, which always checks the "
+          "timestamps."));
+
 ////////////////////////////////////////////////////////////////////
 //     Function: init_libputil
 //  Description: Initializes the library.  This must be called at

panda/src/putil/config_util.h

@@ -48,6 +48,7 @@ extern ConfigVariableDouble sleep_precision;
 
 extern EXPCL_PANDA_PUTIL ConfigVariableBool preload_textures;
 extern EXPCL_PANDA_PUTIL ConfigVariableBool preload_simple_textures;
+extern EXPCL_PANDA_PUTIL ConfigVariableBool cache_check_timestamps;
 
 extern EXPCL_PANDA_PUTIL void init_libputil();
 

panda/src/putil/datagramInputFile.I

@@ -24,6 +24,7 @@ DatagramInputFile() {
   _read_first_datagram = false;
   _in = (istream *)NULL;
   _owns_in = false;
+  _timestamp = 0;
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/putil/datagramInputFile.cxx

@@ -45,6 +45,7 @@ open(const FileReference *file) {
     // No such file.
     return false;
   }
+  _timestamp = _vfile->get_timestamp();
   _in = _vfile->open_read_file(true);
   _owns_in = (_in != (istream *)NULL);
   return _owns_in && !_in->fail();
@@ -66,6 +67,7 @@ open(istream &in, const Filename &filename) {
   _in = ∈
   _owns_in = false;
   _filename = filename;
+  _timestamp = 0;
 
   if (!filename.empty()) {
     _file = new FileReference(filename);
@@ -92,6 +94,7 @@ close() {
 
   _file.clear();
   _filename = Filename();
+  _timestamp = 0;
 
   _read_first_datagram = false;
   _error = false;
@@ -309,6 +312,18 @@ get_filename() {
   return _filename;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: DatagramInputFile::get_timestamp
+//       Access: Published, Virtual
+//  Description: Returns the on-disk timestamp of the file that was
+//               read, at the time it was opened, if that is
+//               available, or 0 if it is not.
+////////////////////////////////////////////////////////////////////
+time_t DatagramInputFile::
+get_timestamp() const {
+  return _timestamp;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: DatagramInputFile::get_file
 //       Access: Published, Virtual

panda/src/putil/datagramInputFile.h

@@ -47,6 +47,7 @@ PUBLISHED:
   virtual bool is_error();
 
   virtual const Filename &get_filename();
+  virtual time_t get_timestamp() const;
   virtual const FileReference *get_file();
   virtual VirtualFile *get_vfile();
   virtual streampos get_file_pos();
@@ -59,6 +60,7 @@ private:
   istream *_in;
   bool _owns_in;
   Filename _filename;
+  time_t _timestamp;
 };
 
 #include "datagramInputFile.I"