fix shutdown order

panda/src/gobj/vertexDataPage.cxx

@@ -53,7 +53,11 @@ ConfigVariableInt max_disk_vertex_data
           "limit."));
 
 PT(VertexDataPage::PageThreadManager) VertexDataPage::_thread_mgr;
-Mutex VertexDataPage::_tlock;
+
+// This is a reference to an allocated Mutex, instead of just a static
+// Mutex, to protect against ordering issues when the application
+// shuts down.
+Mutex &VertexDataPage::_tlock = *(new Mutex("VertexDataPage::_tlock"));
 
 SimpleLru VertexDataPage::_resident_lru("resident", max_resident_vertex_data);
 SimpleLru VertexDataPage::_compressed_lru("compressed", max_compressed_vertex_data);

panda/src/gobj/vertexDataPage.h

@@ -160,7 +160,7 @@ private:
   };
 
   static PT(PageThreadManager) _thread_mgr;
-  static Mutex _tlock;  // Protects _thread_mgr and all of its members.
+  static Mutex &_tlock;  // Protects _thread_mgr and all of its members.
 
   unsigned char *_page_data;
   size_t _size, _allocated_size, _uncompressed_size;

panda/src/pipeline/config_pipeline.cxx

@@ -36,6 +36,17 @@ ConfigVariableBool support_threads
           "does not affect the operation of mutexes and other synchronization "
           "primitives, just the creation of threads."));
 
+ConfigVariableBool name_deleted_mutexes
+("name-deleted-mutexes", false,
+ PRC_DESC("Set this true to allocate a name to each Mutex object that "
+          "destructs, so if the Mutex is locked after destruction, we can "
+          "print out its name to aid debugging.  This is only available "
+          "when compiled with DEBUG_THREADS.  Enabling this variable will "
+          "cause a memory leak, so you should only enable it when you are "
+          "specifically tracking down an operation on a deleted Mutex.  "
+          "It is not guaranteed to work, of course, because the memory "
+          "for a deleted Mutex may become reused for some other purpose."));
+
 ConfigVariableInt thread_stack_size
 ("thread-stack-size", 4194304,
  PRC_DESC("Specifies the minimum size, in bytes, of the stack that will be "

panda/src/pipeline/config_pipeline.h

@@ -26,6 +26,7 @@ NotifyCategoryDecl(pipeline, EXPCL_PANDA_PIPELINE, EXPTP_PANDA_PIPELINE);
 NotifyCategoryDecl(thread, EXPCL_PANDA_PIPELINE, EXPTP_PANDA_PIPELINE);
 
 extern EXPCL_PANDA_PIPELINE ConfigVariableBool support_threads;
+extern ConfigVariableBool name_deleted_mutexes;
 extern ConfigVariableInt thread_stack_size;
 
 extern EXPCL_PANDA_PIPELINE void init_libpipeline();

panda/src/pipeline/mutexDebug.cxx

@@ -32,6 +32,7 @@ MutexDebug(const string &name, bool allow_recursion, bool lightweight) :
   _lightweight(lightweight),
   _locking_thread(NULL),
   _lock_count(0),
+  _deleted_name(NULL),
   _cvar_impl(*get_global_lock())
 {
 #ifndef SIMPLE_THREADS
@@ -50,6 +51,16 @@ MutexDebug::
 ~MutexDebug() {
   nassertv(_locking_thread == NULL && _lock_count == 0);
 
+  // If the config variable says to, allocate (and leak) a string name
+  // for the mutex, so we can report which mutex it is that has
+  // destructed after the fact.
+  if (name_deleted_mutexes) {
+    ostringstream strm;
+    strm << *this;
+    string name = strm.str();
+    _deleted_name = strdup((char *)name.c_str());
+  }
+
   // Put a distinctive, bogus lock count in upon destruction, so we'll
   // be more likely to notice a floating pointer.
   _lock_count = -100;
@@ -83,7 +94,18 @@ void MutexDebug::
 do_lock() {
   // If this assertion is triggered, you tried to lock a
   // recently-destructed mutex.
-  nassertv(_lock_count != -100);
+  nassertd(_lock_count != -100) {
+    pipeline_cat.error()
+      << "Destructed mutex: " << (void *)this << "\n";
+    if (name_deleted_mutexes && _deleted_name != NULL) {
+      pipeline_cat.error()
+        << _deleted_name << "\n";
+    } else {
+      pipeline_cat.error()
+        << "Configure name-deleted-mutexes 1 to see the mutex name.\n";
+    }
+    return;
+  }
 
   Thread *this_thread = Thread::get_current_thread();
 
@@ -189,7 +211,18 @@ void MutexDebug::
 do_release() {
   // If this assertion is triggered, you tried to release a
   // recently-destructed mutex.
-  nassertv(_lock_count != -100);
+  nassertd(_lock_count != -100) {
+    pipeline_cat.error()
+      << "Destructed mutex: " << (void *)this << "\n";
+    if (name_deleted_mutexes && _deleted_name != NULL) {
+      pipeline_cat.error()
+        << _deleted_name << "\n";
+    } else {
+      pipeline_cat.error()
+        << "Configure name-deleted-mutexes 1 to see the mutex name.\n";
+    }
+    return;
+  }
 
   Thread *this_thread = Thread::get_current_thread();
 

panda/src/pipeline/mutexDebug.h

@@ -62,6 +62,7 @@ private:
   bool _lightweight;
   Thread *_locking_thread;
   int _lock_count;
+  char *_deleted_name; // To help I.D. a destructed mutex.
 
   // For _lightweight mutexes.
   typedef pmap<Thread *, int> MissedThreads;