Shift file handling in importer to a standardized function

Added santizing of strings to filenames, object names and asset names for the importer process
Added sanitizeString console function for above
Added processing of terrainBlock object's fields for importer
Added check to avoid updateTSShapeLoadProgress spam
Adjusted folderPrefix logic to walk up directory to find assetName that isn't already in use.

Engine/source/T3D/assets/assetImporter.cpp

@@ -2381,11 +2381,25 @@ void AssetImporter::resolveAssetItemIssues(AssetImportObject* assetItem)
       }
       else if (activeImportConfig->DuplicateAutoResolution == String("FolderPrefix"))
       {
-         //Set trailing number
          String renamedAssetName = assetItem->assetName;
-         String owningFolder = assetItem->filePath.getDirectory(assetItem->filePath.getDirectoryCount() - 1);
 
-         renamedAssetName = owningFolder + "_" + renamedAssetName;
+         //Set trailing number
+         S32 dirIndex = assetItem->filePath.getDirectoryCount() - 1;
+         while (dirIndex > -1)
+         {
+            renamedAssetName = assetItem->assetName;
+            String owningFolder = assetItem->filePath.getDirectory(dirIndex);
+
+            renamedAssetName = owningFolder + "_" + renamedAssetName;
+
+            if (AssetDatabase.isDeclaredAsset(renamedAssetName))
+            {
+               dirIndex--;
+               continue;
+            }
+
+            break;
+         }
 
          //Log it's renaming
          dSprintf(importLogBuffer, sizeof(importLogBuffer), "Asset %s was renamed due to %s as part of the Import Configuration", assetItem->assetName.c_str(), humanReadableReason.c_str());

Engine/source/console/consoleFunctions.cpp

@@ -574,6 +574,40 @@ DefineEngineFunction( stripChars, const char*, ( const char* str, const char* ch
    return( ret );
 }
 
+//-----------------------------------------------------------------------------
+DefineEngineFunction(sanitizeString, const char*, (const char* str), ,
+   "Sanitizes a string of common name issues, such as:\n"
+   "Starting with numbers, replacing spaces with _, and removing any name un-compliant characters such as .,- etc\n"
+   "@param str The string to sanitize.\n"
+   "@return A version of @a str with all occurrences of invalid characters removed.\n\n"
+   "@tsexample\n"
+   "cleanString( \"123 .-_abc\"); // Returns \"__abc\"."
+   "@endtsexample\n"
+   "@ingroup Strings")
+{
+   String processedString = str;
+
+   U32 start;
+   U32 end;
+   String firstNumber = String::GetFirstNumber(processedString, start, end);
+   if (!firstNumber.isEmpty() && processedString.startsWith(firstNumber.c_str()))
+      processedString = processedString.replace(firstNumber, "");
+
+   processedString = processedString.replace(" ", "_");
+
+   U32 len = processedString.length() + 1;
+   char* ret = Con::getReturnBuffer(len);
+   dStrcpy(ret, processedString.c_str(), len);
+
+   U32 pos = dStrcspn(ret, "-+*/%$&�=()[].?\\\"#,;!~<>|�^{}");
+   while (pos < dStrlen(ret))
+   {
+      dStrcpy(ret + pos, ret + pos + 1, len - pos);
+      pos = dStrcspn(ret, "-+*/%$&�=()[].?\\\"#,;!~<>|�^{}");
+   }
+   return(ret);
+}
+
 //-----------------------------------------------------------------------------
 
 DefineEngineFunction( strlwr, const char*, ( const char* str ),,

Engine/source/ts/loader/tsShapeLoader.cpp

@@ -125,7 +125,8 @@ void TSShapeLoader::updateProgress(S32 major, const char* msg, S32 numMinor, S32
       progressMsg = avar("%s (%d of %d)", msg, minor + 1, numMinor);
    }
 
-   Con::executef("updateTSShapeLoadProgress", Con::getFloatArg(progress), progressMsg);
+   if(Con::isFunction("updateTSShapeLoadProgress"))
+      Con::executef("updateTSShapeLoadProgress", Con::getFloatArg(progress), progressMsg);
 }
 
 //-----------------------------------------------------------------------------
@@ -1353,4 +1354,4 @@ DefineEngineFunction( getFormatFilters, const char*, ( ),,
 DefineEngineFunction(isSupportedFormat, bool, (const char* extension), , "")
 {
    return TSShapeLoader::isSupportedFormat(extension);
-}
+}

Templates/BaseGame/game/tools/projectImporter/scripts/pre40/T3Dpre4ProjectImporter.tscript

@@ -77,8 +77,15 @@ function T3Dpre4ProjectImporter::copyFiles(%this)
    {
       %filePath = filePath(%file);
       %fileName = fileName(%file);
+      %fileBase = fileBase(%file);
       %fileExt = fileExt(%file);
       
+      if(endsWith(%fileName, ".asset.taml"))
+      {
+         %fileBase = strreplace(%fileBase, ".asset", "");
+         %fileExt = ".asset.taml";
+      }
+      
       if(%fileExt $= ".dll" || %fileExt $= ".log" || %fileExt $= ".exe" || %fileExt $= ".manifest"|| %fileExt $= ".h" ||
          %fileExt $= ".cpp" || %fileExt $= ".so" || %fileExt $= ".do" || %fileExt $= ".lib" ||%fileExt $= ".exp")
       {
@@ -99,6 +106,17 @@ function T3Dpre4ProjectImporter::copyFiles(%this)
       }
       
       %targetFilePath = strReplace(%file, $ProjectImporter::sourceContentFolder, $ProjectImporter::modulePath);
+      
+      %sanitizedFilename = sanitizeString(%fileBase);
+      if(startsWith(%sanitizedFilename, "_"))
+      {
+         %sanitizedFilename = substr(%sanitizedFilename, 1, -1);
+      }
+      if(%sanitizedFilename !$= %fileBase)
+      {
+         %targetFilePath = filePath(%targetFilePath) @ "/" @ %sanitizedFilename @ %fileExt;
+      }
+      
       %targetFolder = filePath(%targetFilePath);
       
       if(!isDirectory(%targetFolder))
@@ -214,7 +232,7 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this)
             %line = $ProjectImporter::fileObject.readLine();
             %trimmedLine = trim(%line);
             
-            if(strIsMatchExpr("*new*(*)*", %line))
+            if(strIsMatchExpr("*new*(*)*", %line) && strpos(%line, "::") == -1)
             {
                %className = findObjectClass(%line, "new");
 
@@ -246,6 +264,18 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this)
                   %fileWasChanged = true;
                }
                   
+               %sanitizedName = sanitizeString(%objectName);
+               if(startsWith(%sanitizedName, "_"))
+               {
+                  %sanitizedName = substr(%sanitizedName, 1, -1);
+               }
+               if(%sanitizedName !$= %objectName)
+               {
+                  %line = strReplace(%line, %objectName, %sanitizedName);
+                  
+                  %fileWasChanged = true;
+               }
+                  
                   if(%objectClassStack.count() == 1)
                   {
                   %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -278,6 +308,18 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this)
                
                %objectName = findObjectName(%line, "singleton");
                   
+               %sanitizedName = sanitizeString(%objectName);
+               if(startsWith(%sanitizedName, "_"))
+               {
+                  %sanitizedName = substr(%sanitizedName, 1, -1);
+               }
+               if(%sanitizedName !$= %objectName)
+               {
+                  %line = strReplace(%line, %objectName, %sanitizedName);
+                  
+                  %fileWasChanged = true;
+               }
+               
                   if(%objectClassStack.count() == 1)
                   {
                   %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -418,7 +460,7 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this)
             %line = $ProjectImporter::fileObject.readLine();
             %trimmedLine = trim(%line);
             
-            if(strIsMatchExpr("*new*(*)*", %line))
+            if(strIsMatchExpr("*new*(*)*", %line) && strpos(%line, "::") == -1)
             {
                %className = findObjectClass(%line, "new");
                
@@ -428,6 +470,18 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this)
                
                if(%objectName !$= "")
                {
+                  %sanitizedName = sanitizeString(%objectName);
+                  if(startsWith(%sanitizedName, "_"))
+                  {
+                     %sanitizedName = substr(%sanitizedName, 1, -1);
+                  }
+                  if(%sanitizedName !$= %objectName)
+                  {
+                     %line = strReplace(%line, %objectName, %sanitizedName);
+                     
+                     %fileWasChanged = true;
+                  }
+                  
                   if(%objectClassStack.count() == 1)
                   {
                      %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -513,6 +567,18 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this)
                
                if(%objectName !$= "")
                {
+                  %sanitizedName = sanitizeString(%objectName);
+                  if(startsWith(%sanitizedName, "_"))
+                  {
+                     %sanitizedName = substr(%sanitizedName, 1, -1);
+                  }
+                  if(%sanitizedName !$= %objectName)
+                  {
+                     %line = strReplace(%line, %objectName, %sanitizedName);
+                     
+                     %fileWasChanged = true;
+                  }
+               
                   if(%objectClassStack.count() == 1)
                   {
                      %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -764,6 +830,7 @@ T3Dpre4ProjectImporter::genProcessor("GroundPlane", "material materialAsset");
 T3Dpre4ProjectImporter::genProcessor("LevelInfo", "accuTexture accuTextureAsset");
 T3Dpre4ProjectImporter::genProcessor("TSStatic", "shape shapeAsset shapeName shapeAsset");
 T3Dpre4ProjectImporter::genProcessor("TSForestItemData", "shape shapeAsset shapeName shapeAsset shapeFile shapeAsset");
+T3Dpre4ProjectImporter::genProcessor("TerrainBlock", "terrainFile terrainAsset");
 //==============================================================================
 // Levels
 //==============================================================================
@@ -1034,57 +1101,8 @@ function processGuiBitmapButtonCtrlField(%line, %originalFieldName, %newFieldNam
          //These are going to be texture/render targets, and we can leave them alone
          return %line;
       }
-      //find any assets with that filename
-      else if(startsWith(%value, "./"))
-      {
-         %targetFilename = strReplace(%value, "./", $ProjectImporter::currentFilePath @ "/");
-      }
-      else if(startsWith(%value, "../"))
-      {
-         %slashPos = strposr($ProjectImporter::currentFilePath, "/");
-         if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
-         {
-            %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
-         }
-         
-         %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
-         %targetFilename = strReplace(%value, "../", %parentPath @ "/");
-      }
-      else if(startsWith(%value, "~"))
-      {
-         %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/");
-         if(!isFile(%targetFilename))
-         {
-            %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/main/");
-         }
-      }
-      else if ((strpos(%value,"/") == -1)&&(strpos(%value,"\\") == -1))
-      {
-         %targetFilename = $ProjectImporter::currentFilePath @ %value;
-      }
-      else if(!startsWith(%value, $ProjectImporter::modulePath @ "/"))
-      {
-         %targetFilename = $ProjectImporter::modulePath @ "/" @ %value;
-      }
-      else
-      {
-         %targetFilename = %value;
-      }
-      
-      %targetFilename = strReplace(%targetFilename, "//", "/");
-      %targetFilename = testFilenameExtensions(%targetFilename);
       
-      if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out
-      {
-         //Now we test for a suffix
-         %targetPath = filePath(%targetFilename);
-         %targetName = fileBase(%targetFilename);
-         %targetExt = fileExt(%targetFilename);
-         
-         %targetFilename = %targetPath @ "/" @ %targetName @ "_n" @ %targetExt;
-         %targetFilename = strReplace(%targetFilename, "//", "/");
-         %targetFilename = testFilenameExtensions(%targetFilename);
-      }
+      %targetFilename = sanitizeFilename(%value);
       
       //If we still have nothing, then we fail it out
       if(!isFile(%targetFilename))

Templates/BaseGame/game/tools/projectImporter/scripts/projectImporter.tscript

@@ -220,10 +220,7 @@ function ProjectImportWizardPage3::openPage(%this)
          %topFolder = getToken($ProjectImporter::sourceContentFolder, "/", %slashCount-2);
          
       //clean up invalid characters and stuff
-      %topFolder = strReplace(%topFolder, " ", "");
-      %topFolder = strReplace(%topFolder, "!", "");
-      %topFolder = strReplace(%topFolder, "-", "");
-      %topFolder = strReplace(%topFolder, ".", "");
+      %topFolder = sanitizeString(%topFolder);
      
       $ProjectImporter::useExistingModule = false;
       $ProjectImporter::moduleName = %topFolder; //preseed the module name
@@ -366,6 +363,91 @@ function beginProjectImport()
    AssetBrowser.refresh(); //update the AB just in case
 }
 
+function sanitizeFilename(%file)
+{
+   if(startsWith(%file, "./"))
+   {
+      %targetFilename = strReplace(%file, "./", $ProjectImporter::currentFilePath @ "/");
+   }
+   else if(startsWith(%file, "../"))
+   {
+      %slashPos = strposr($ProjectImporter::currentFilePath, "/");
+      if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
+      {
+         %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
+      }
+      
+      %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
+      %targetFilename = strReplace(%file, "../", %parentPath @ "/");
+   }
+   else if(startsWith(%file, "~"))
+   {
+      %targetFilename = strReplace(%file, "~", $ProjectImporter::modulePath @ "/");
+      if(!isFile(%targetFilename))
+      {
+         %targetFilename = strReplace(%file, "~", $ProjectImporter::modulePath @ "/main/");
+      }
+   }
+   else if ((strpos(%file,"/") == -1)&&(strpos(%file,"\\") == -1))
+   {
+      %targetFilename = $ProjectImporter::currentFilePath @ %file;
+   }
+   else if(!startsWith(%file, $ProjectImporter::modulePath @ "/"))
+   {
+      %targetFilename = $ProjectImporter::modulePath @ "/" @ %file;
+   }
+   else
+   {
+      %targetFilename = %file;
+   }
+   
+   //Now we test for a suffix
+   %flName = fileName(%targetFilename);
+   %targetPath = filePath(%targetFilename);
+   %targetName = fileBase(%targetFilename);
+   %targetExt = fileExt(%targetFilename);
+   
+   if(endsWith(%flName, ".asset.taml"))
+   {
+      %targetName = strreplace(%targetName, ".asset", "");
+      %targetExt = ".asset.taml";
+   }
+   
+   %sanitizedName = sanitizeString(%targetName);
+   if(startsWith(%sanitizedName, "_"))
+   {
+      %sanitizedName = substr(%sanitizedName, 1, -1);
+   }
+   if(%sanitizedName !$= %targetName)
+   {
+      %targetName = %sanitizedName;
+      %targetFilename = %targetPath @ "/" @ %targetName @ %targetExt;
+   }
+   
+   if(!isFile(%targetFilename))
+   {
+      %bitmapFile = %targetPath @ "/" @ %targetName @ "_n" @ %targetExt;
+      if(isFile(%bitmapFile))
+         %targetFilename = %bitmapFile;
+   }
+   
+   if(!isFile(%targetFilename))
+   {
+      %targetFilename = testFilenameExtensions(%targetFilename);
+   }
+   
+   %targetFilename = strReplace(%targetFilename, "//", "/");
+   
+   if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out
+   {
+      return %file;
+   }
+   else
+   {
+      return %targetFilename;
+   }
+}
+
 function testFilenameExtensions(%filename)
 {
    %ext = fileExt(%filename);  
@@ -432,47 +514,17 @@ function processLegacyField(%line, %originalFieldName, %newFieldName)
          //These are going to be texture/render targets, and we can leave them alone
          return %line;
       }
-      //find any assets with that filename
-      else if(startsWith(%value, "./"))
-      {
-         %targetFilename = strReplace(%value, "./", $ProjectImporter::currentFilePath @ "/");
-      }
-      else if(startsWith(%value, "../"))
-      {
-         %slashPos = strposr($ProjectImporter::currentFilePath, "/");
-         if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
-         {
-            %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
-         }
          
-         %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
-         %targetFilename = strReplace(%value, "../", %parentPath @ "/");
-      }
-      else if(startsWith(%value, "~"))
-      {
-         %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/");
-         if(!isFile(%targetFilename))
+      %targetFilename = sanitizeFilename(%value);
+      
+      if(isObject(%targetFilename))
          {
-            %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/main/");
-         }
-      }
-      else if ((strpos(%value,"/") == -1)&&(strpos(%value,"\\") == -1))
-      {
-         %targetFilename = $ProjectImporter::currentFilePath @ %value;
-      }
-      else if(!startsWith(%value, $ProjectImporter::modulePath @ "/"))
-      {
-         %targetFilename = $ProjectImporter::modulePath @ "/" @ %value;
+         //likely a material name, so handle it that way
+         %assetId = MaterialAsset::getAssetIdByMaterialName(%targetFilename);
       }
       else
       {
-         %targetFilename = %value;
-      }
-      
-      %targetFilename = strReplace(%targetFilename, "//", "/");
-      %targetFilename = testFilenameExtensions(%targetFilename);
-      
-      if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out
+         if(!isFile(%targetFilename))
       {
          error("Legacy Project Importer - file described in line could not be found/is not valid");
          return %line;
@@ -483,11 +535,12 @@ function processLegacyField(%line, %originalFieldName, %newFieldName)
       if(%foundAssets != 0)
       {
          %assetId = $ProjectImporter::assetQuery.getAsset(0);
-         echo("Legacy Project Importer - processing of legacy field line's value: " @ %value @ " has found a matching AssetId: " @ %assetId);
+         }
       }
      
       if(%assetId !$= "" && AssetDatabase.isDeclaredAsset(%assetId))
       {
+         echo("Legacy Project Importer - processing of legacy field line's value: " @ %value @ " has found a matching AssetId: " @ %assetId);
          //if (%assetId.getStatusString() $= "Ok")
          %outLine = strReplace(%outLine, %value, %assetId);
          //else
@@ -523,45 +576,7 @@ function processLegacyShapeConstructorField(%line)
    if(strPos(%animSourcePath, ":") != -1)
       return %line;
                
-   //otherwise, try and see if we've got an animation source file here
-   if(startsWith(%animSourcePath, "./"))
-   {
-      %targetFilename = strReplace(%animSourcePath, "./", $ProjectImporter::currentFilePath @ "/");
-   }
-   else if(startsWith(%animSourcePath, "../"))
-   {
-      %slashPos = strposr($ProjectImporter::currentFilePath, "/");
-      if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
-      {
-         %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
-      }
-      
-      %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
-      %targetFilename = strReplace(%animSourcePath, "../", %parentPath @ "/");
-   }
-   else if(startsWith(%animSourcePath, "~"))
-   {
-      %targetFilename = strReplace(%animSourcePath, "~", $ProjectImporter::modulePath @ "/");
-      if(!isFile(%targetFilename))
-      {
-         %targetFilename = strReplace(%animSourcePath, "~", $ProjectImporter::modulePath @ "/main/");
-      }
-   }
-   else if ((strpos(%animSourcePath,"/") == -1)&&(strpos(%animSourcePath,"\\") == -1))
-   {
-      %targetFilename = $ProjectImporter::currentFilePath @ %animSourcePath;
-   }
-   else if(!startsWith(%animSourcePath, $ProjectImporter::modulePath @ "/"))
-   {
-      %targetFilename = $ProjectImporter::modulePath @ "/" @ %animSourcePath;
-   }
-   else
-   {
-      %targetFilename = %animSourcePath;
-   }
-   
-   %targetFilename = strReplace(%targetFilename, "//", "/");
-   %targetFilename = testFilenameExtensions(%targetFilename);
+   %targetFilename = sanitizeFilename(%animSourcePath);
    
    if(!isFile(%targetFilename))
    {
@@ -828,6 +843,14 @@ function beginTerrainImport()
       %filePath = filePath(%file);
       if(%fileExt $= ".ter")
       {
+         %sanitizedFile = sanitizeFilename(%file);
+         if(%sanitizedFile !$= %file)
+         {
+            %file = %sanitizedFile;
+            %fileName = fileName(%file); 
+            %filePath = filePath(%file);
+         }
+            
          $ProjectImporter::assetQuery.clear();
          %assetsFound = AssetDatabase.findAssetLooseFile($ProjectImporter::assetQuery, %file);
          if(%assetsFound == 0)