datablock caching, and dependency-object vs hooked up class method segregation

1) relies on https://github.com/Areloch/Torque3D/pull/99 for modular resolvers (re-included for ease of testing)
2) adds a new module::onServerScriptExec(%this) callback executed after datablock transmission
3) bypasses DB transmission in favor of a straight file read if the resulting network data would (via CRC compare) match what a client already has saved off under data\cache\<client or server>

Templates/BaseGame/game/core/clientServer/scripts/client/levelDownload.cs

@@ -38,7 +38,73 @@
 //----------------------------------------------------------------------------
 // Phase 1 
 //----------------------------------------------------------------------------
-function clientCmdMissionStartPhase1(%seq, %missionName)
+$pref::Client::EnableDatablockCache = true;
+$pref::Client::DatablockCacheFilename = "data/cache/client/datablock_cache_c.dbc";
+
+function clientCmdMissionStartPhase1_LoadCache(%seq, %missionName)
+{
+  if ($pref::Client::EnableDatablockCache && $loadFromDatablockCache)
+  {
+    if (!$pref::Video::disableVerticalSync)
+    {
+      warn("Disabling Vertical Sync during datablock cache load to avoid significant slowdown.");
+      $AFX_tempDisableVSync = true;
+
+      $pref::Video::disableVerticalSync = true;
+      Canvas.resetVideoMode();
+    }
+
+    echo("<<<< Loading Datablocks From Cache >>>>");
+    if (ServerConnection.loadDatablockCache_Begin())
+    {
+      schedule(10, 0, "updateLoadDatablockCacheProgress", %seq, %missionName);
+    }
+  }
+}
+
+function updateLoadDatablockCacheProgress(%seq, %missionName)
+{
+   if (ServerConnection.loadDatablockCache_Continue())
+   {
+      $loadDatablockCacheProgressThread = schedule(10, 0, "updateLoadDatablockCacheProgress", %seq, %missionName);
+      return;
+   }
+ 
+   if ($AFX_tempDisableVSync)
+   {
+     warn("Restoring Vertical Sync setting.");
+     $AFX_tempDisableVSync = false;
+
+     $pref::Video::disableVerticalSync = false;
+     Canvas.resetVideoMode();
+   }
+
+   echo("<<<< Finished Loading Datablocks From Cache >>>>");
+   clientCmdMissionStartPhase2(%seq,%missionName);
+}
+
+function updateLoadDatablockCacheProgress(%seq, %missionName)
+{
+   if (ServerConnection.loadDatablockCache_Continue())
+   {
+      $loadDatablockCacheProgressThread = schedule(10, 0, "updateLoadDatablockCacheProgress", %seq, %missionName);
+      return;
+   }
+ 
+   if ($AFX_tempDisableVSync)
+   {
+     warn("Restoring Vertical Sync setting.");
+     $AFX_tempDisableVSync = false;
+
+     $pref::Video::disableVerticalSync = false;
+     Canvas.resetVideoMode();
+   }
+
+   echo("<<<< Finished Loading Datablocks From Cache >>>>");
+   clientCmdMissionStartPhase2(%seq,%missionName);
+}
+
+function clientCmdMissionStartPhase1(%seq, %missionName, %cache_crc)
 {
    // These need to come after the cls.
    echo ("*** New Mission: " @ %missionName);
@@ -61,6 +127,56 @@ function clientCmdMissionStartPhase1(%seq, %missionName)
       PostFXManager::settingsApplyDefaultPreset();
    }
    
+  $loadFromDatablockCache = false;
+  if ($pref::Client::EnableDatablockCache)
+  {
+    %cache_filename = $pref::Client::DatablockCacheFilename;
+
+    // if cache CRC is provided, check for validity
+    if (%cache_crc !$= "")
+    {
+      // check for existence of cache file
+      if (isFile(%cache_filename))
+      { 
+        // here we are not comparing the CRC of the cache itself, but the CRC of
+        // the server cache (stored in the header) when these datablocks were
+        // transmitted.
+        %my_cache_crc = extractDatablockCacheCRC(%cache_filename);
+        echo("<<<< client cache CRC:" SPC %my_cache_crc SPC ">>>>");
+        echo("<<<< comparing CRC codes:" SPC "s:" @ %cache_crc SPC "c:" @ %my_cache_crc SPC ">>>>");
+        if (%my_cache_crc == %cache_crc)
+        {
+          echo("<<<< cache CRC codes match, datablocks will be loaded from local cache. >>>>");
+          $loadFromDatablockCache = true;
+        }
+        else
+        {
+          echo("<<<< cache CRC codes differ, datablocks will be transmitted and cached. >>>>" SPC %cache_crc);
+          setDatablockCacheCRC(%cache_crc);
+        }
+      }
+      else
+      {
+        echo("<<<< client datablock cache does not exist, datablocks will be transmitted and cached. >>>>");
+        setDatablockCacheCRC(%cache_crc);
+      }
+    }
+    else
+    {
+      echo("<<<< server datablock caching is disabled, datablocks will be transmitted. >>>>");
+    }
+    if ($loadFromDatablockCache)
+    {
+      // skip datablock transmission and initiate a cache load
+      commandToServer('MissionStartPhase1Ack_UseCache', %seq);
+      return;
+    }
+  }
+  else if (%cache_crc !$= "")
+  {
+    echo("<<<< client datablock caching is disabled, datablocks will be transmitted. >>>>");
+  }
+  
    onMissionDownloadPhase("LOADING DATABLOCKS");
    
    commandToServer('MissionStartPhase1Ack', %seq);
@@ -164,7 +280,7 @@ function connect(%server)
 {
    %conn = new GameConnection(ServerConnection);
    RootGroup.add(ServerConnection);
-   %conn.setConnectArgs($pref::Player::Name);
+   %conn.setConnectArgs($pref::Player::Name, $ConncetInfoKey);
    %conn.setJoinPassword($Client::Password);
    %conn.connect(%server);
 }

Templates/BaseGame/game/core/clientServer/scripts/server/defaults.cs

@@ -46,8 +46,6 @@ $Pref::Server::ConnectionError =
 // overrides pref::net::port for dedicated servers
 $Pref::Server::Port = 28000;
 
-$Pref::Server::EnableDatablockCache = true;
-$Pref::Server::DatablockCacheFilename = "core/clientServer/scripts/server/afx/cache/afx_datablock_cache.dbc";
 
 // If the password is set, clients must provide it in order
 // to connect to the server

Templates/BaseGame/game/core/clientServer/scripts/server/levelDownload.cs

@@ -38,8 +38,26 @@
 //----------------------------------------------------------------------------
 // Phase 1 
 //----------------------------------------------------------------------------
+$Pref::Server::EnableDatablockCache = true;
+$pref::Server::DatablockCacheFilename = "data/cache/server/datablock_cache_c.dbc";
 function GameConnection::loadMission(%this)
 {
+  %cache_crc = "";
+
+  if ($Pref::Server::EnableDatablockCache)
+  {
+    if (!isDatablockCacheSaved())
+    {
+      echo("<<<< saving server datablock cache >>>>");
+      %this.saveDatablockCache();
+    }
+
+    if (isFile($Pref::Server::DatablockCacheFilename))
+    {
+      %cache_crc = getDatablockCacheCRC();
+      echo("    <<<< sending CRC to client:" SPC %cache_crc SPC ">>>>");
+    }
+  }
    // Send over the information that will display the server info
    // when we learn it got there, we'll send the data blocks
    %this.currentPhase = 0;
@@ -50,12 +68,41 @@ function GameConnection::loadMission(%this)
    }
    else
    {
-      commandToClient(%this, 'MissionStartPhase1', $missionSequence, $Server::MissionFile);
+      commandToClient(%this, 'MissionStartPhase1', $missionSequence, $Server::MissionFile, %cache_crc);
          
       echo("*** Sending mission load to client: " @ $Server::MissionFile);
    }
 }
 
+function serverCmdMissionStartPhase1Ack_UseCache(%client, %seq)
+{
+  echo("<<<< client will load datablocks from a cache >>>>");
+  echo("    <<<< skipping datablock transmission >>>>");
+
+  // Make sure to ignore calls from a previous mission load
+  if (%seq != $missionSequence || !$MissionRunning)
+    return;
+  if (%client.currentPhase != 0)
+    return;
+  %client.currentPhase = 1;
+
+  // Start with the CRC
+  %client.setMissionCRC( $missionCRC );
+
+  %client.onBeginDatablockCacheLoad($missionSequence);
+}
+
+function GameConnection::onBeginDatablockCacheLoad( %this, %missionSequence )
+{
+   // Make sure to ignore calls from a previous mission load
+   if (%missionSequence != $missionSequence)
+      return;
+   if (%this.currentPhase != 1)
+      return;
+   %this.currentPhase = 1.5;
+   commandToClient(%this, 'MissionStartPhase1_LoadCache', $missionSequence, $Server::MissionFile);
+}
+
 function serverCmdMissionStartPhase1Ack(%client, %seq)
 {
    // Make sure to ignore calls from a previous mission load
@@ -157,14 +204,6 @@ function serverCmdMissionStartPhase3Ack(%client, %seq)
       // Set the control object to the default camera
       if (!isObject(%client.camera))
       {
-         if(!isObject(Observer))
-         {
-            datablock CameraData(Observer)
-            {
-               mode = "Observer";
-            };  
-         }
-         
          //if (isDefined("$Game::DefaultCameraClass"))
             %client.camera = spawnObject("Camera", Observer);
       }

Templates/BaseGame/game/core/clientServer/scripts/server/server.cs

@@ -204,10 +204,11 @@ function onServerCreated()
    
    loadDatablockFiles( DatablockFilesList, true );
    
+   callOnModules("onServerScriptExec", "Core");
+   callOnModules("onServerScriptExec", "Game");   
+   
    // Keep track of when the game started
    $Game::StartTime = $Sim::Time;
-
-   onServerCreatedAFX();
 }
 
 /// Shut down the server
@@ -283,6 +284,9 @@ function onServerDestroyed()
    MissionCleanup.delete();
    
    clearServerPaths();
+   
+  if ($Pref::Server::EnableDatablockCache)
+    resetDatablockCache();
 }
 
 /// Guid list maintenance functions

Templates/BaseGame/game/core/utility/scripts/module.cs

@@ -1,5 +1,12 @@
+$traceModuleCalls=false;
+$reportModuleFileConflicts=true;
+if (!isObject(ExecFilesList))
+   new ArrayObject(ExecFilesList);
+  
 function callOnModules(%functionName, %moduleGroup)
 {
+   //clear per module group file execution chain
+   ExecFilesList.empty();
    //Get our modules so we can exec any specific client-side loading/handling
    %modulesList = ModuleDatabase.findModules(false);
    for(%i=0; %i < getWordCount(%modulesList); %i++)
@@ -16,7 +23,14 @@ function callOnModules(%functionName, %moduleGroup)
       {
          eval(%module.scopeSet @ "." @ %functionName @ "();");
       }
-   }   
+   }
+   
+   %execFilecount = ExecFilesList.count();
+   for (%i=0;%i<%execFilecount;%i++)
+   {
+        %filename = ExecFilesList.getKey(%i);
+        exec(%filename);
+   }
 }
 
 function loadModuleMaterials(%moduleGroup)
@@ -69,10 +83,12 @@ function SimSet::getModulePath(%scopeSet)
    return "";
 }
 
-function SimSet::registerDatablock(%scopeSet, %datablockFilePath)
+function SimSet::registerDatablock(%scopeSet, %datablockFilePath, %isExclusive)
 {
+   if ($traceModuleCalls)
+      warn("SimSet::registerDatablock");
    %name = %scopeSet.getName();
-   %moduleDef = ModuleDatabase.findModule(%name);
+   %moduleDef = ModuleDatabase.findModule(%name, 1);
      
    if(!isObject(%moduleDef))
    {
@@ -89,6 +105,176 @@ function SimSet::registerDatablock(%scopeSet, %datablockFilePath)
    %relativePath = makeRelativePath(%datablockFilePath);
    
    %fullPath = pathConcat(%moduleDef.ModulePath, %relativePath);
+   ///go through all entries
+   %locked = false;
+   %dbFilecount = DatablockFilesList.count();
+   for (%i=0;%i<%dbFilecount;%i++)
+   {
+        %check = DatablockFilesList.getKey(%i);
+        //look for a substring match
+        %isMatch = strIsMatchExpr("*"@ %datablockFilePath,%check );
+        if (%isMatch)
+        {
+            //check if we're already locked in
+            //and kill off any duplicates
+            //do note that doing it in this order means setting exclusive twice
+            //allows one to override exclusive with exclusive
+            %locked = DatablockFilesList.getValue(%i);
+
+            if ((!%locked && !%isExclusive)&&($reportModuleFileConflicts))
+                error("found" SPC %datablockFilePath SPC "duplicate file!");
+            if (!%locked || (%locked && %isExclusive))
+            {
+                DatablockFilesList.erase(%i);
+            }
+        }
+   }
+   //if we're not locked, or we are exclusive, go ahead and add it to the pile
+   //(ensures exclusives get re-added after that erasure)
+   if (!%locked || %isExclusive)
+       DatablockFilesList.add(%fullPath,%isExclusive);
+   if ($traceModuleCalls)
+      DatablockFilesList.echo();
+}
+
+function SimSet::unRegisterDatablock(%scopeSet, %datablockFilePath)
+{
+   if ($traceModuleCalls)
+      warn("SimSet::unRegisterDatablock");
+   %name = %scopeSet.getName();
+   %moduleDef = ModuleDatabase.findModule(%name, 1);
+     
+   if(!isObject(%moduleDef))
+   {
+      error("Module::unRegisterDatablock() - unable to find a module with the moduleID of " @ %name);
+      return;
+   }
+   
+   if(!isObject(DatablockFilesList))
+   {
+      error("Module::unRegisterDatablock() - DatablockFilesList array object doesn't exist!");
+      return;
+   }
+   
+   %relativePath = makeRelativePath(%datablockFilePath);
    
-   DatablockFilesList.add(%fullPath);
+   %fullPath = pathConcat(%moduleDef.ModulePath, %relativePath);
+   ///go through all entries
+   %locked = false;
+   %dbFilecount = DatablockFilesList.count();
+   for (%i=0;%i<%dbFilecount;%i++)
+   {
+        %check = DatablockFilesList.getKey(%i);
+        //look for a substring match
+        %isMatch = strIsMatchExpr("*"@ %datablockFilePath,%check );
+        if (%isMatch)
+        {
+            //check if we're already locked in. if not, kill it.
+            %locked = DatablockFilesList.getValue(%i);
+            if (!%locked)
+            {
+                DatablockFilesList.erase(%i);
+            }
+        }
+   }
+   if ($traceModuleCalls)
+      DatablockFilesList.echo();
+}
+
+function SimSet::queueExec(%scopeSet, %execFilePath, %isExclusive)
+{
+   if ($traceModuleCalls)
+      warn("SimSet::queueExec");
+   %name = %scopeSet.getName();
+   %moduleDef = ModuleDatabase.findModule(%name, 1);
+     
+   if(!isObject(%moduleDef))
+   {
+      error("Module::queueExec() - unable to find a module with the moduleID of " @ %name);
+      return;
+   }
+   
+   if(!isObject(ExecFilesList))
+   {
+      error("Module::queueExec() - ExecFilesList array object doesn't exist!");
+      return;
+   }
+   
+   if ($traceModuleCalls)
+      warn("module root path="@ makeRelativePath(%moduleDef.ModulePath));
+  
+   %fullPath = makeRelativePath(%moduleDef.ModulePath) @ %execFilePath;
+   ///go through all entries
+   %locked = false;
+   %execFilecount = ExecFilesList.count();
+   for (%i=0;%i<%execFilecount;%i++)
+   {
+        %check = ExecFilesList.getKey(%i);
+        //look for a substring match
+        %isMatch = strIsMatchExpr("*"@ %execFilePath,%check );
+        if (%isMatch)
+        {
+            //check if we're already locked in
+            //and kill off any duplicates
+            //do note that doing it in this order means setting exclusive twice
+            //allows one to override exclusive with exclusive
+            %locked = ExecFilesList.getValue(%i);
+            if ((!%locked && !%isExclusive)&&($reportModuleFileConflicts))
+                error("found" SPC %execFilePath SPC "duplicate file!");
+            if (!%locked || (%locked && %isExclusive))
+            {
+                ExecFilesList.erase(%i);
+            }
+        }
+   }
+   //if we're not locked, or we are exclusive, go ahead and add it to the pile
+   //(ensures exclusives get re-added after that erasure)
+   if (!%locked || %isExclusive)
+       ExecFilesList.add(%fullPath,%isExclusive);
+   if ($traceModuleCalls)       
+      ExecFilesList.echo();
+}
+
+function SimSet::unQueueExec(%scopeSet, %execFilePath)
+{
+   if ($traceModuleCalls)
+      warn("SimSet::unRegisterDatablock");
+   %name = %scopeSet.getName();
+   %moduleDef = ModuleDatabase.findModule(%name, 1);
+     
+   if(!isObject(%moduleDef))
+   {
+      error("Module::unRegisterDatablock() - unable to find a module with the moduleID of " @ %name);
+      return;
+   }
+   
+   if(!isObject(ExecFilesList))
+   {
+      error("Module::unRegisterDatablock() - ExecFilesList array object doesn't exist!");
+      return;
+   }
+   
+   %relativePath = makeRelativePath(%execFilePath);
+   
+   %fullPath = pathConcat(%moduleDef.ModulePath, %relativePath);
+   ///go through all entries
+   %locked = false;
+   %execFilecount = ExecFilesList.count();
+   for (%i=0;%i<%execFilecount;%i++)
+   {
+        %check = ExecFilesList.getKey(%i);
+        //look for a substring match
+        %isMatch = strIsMatchExpr("*"@ %execFilePath,%check );
+        if (%isMatch)
+        {
+            //check if we're already locked in. if not, kill it.
+            %locked = ExecFilesList.getValue(%i);
+            if (!%locked)
+            {
+                ExecFilesList.erase(%i);
+            }
+        }
+   }
+   if ($traceModuleCalls)
+      ExecFilesList.echo();
 }