o3de/Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyComponentConverter.py

"""
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.

SPDX-License-Identifier: Apache-2.0 OR MIT



Lumberyard Legacy Renderer to Atom Component Conversion Script


What does this script do?
================================================
This script walks through all the .slice, .layer, .ly, and .cry files in a
project and attempts to convert the following components into something
reasonably similar that renders in Atom:
Mesh
Material
PointLight
Actor


Do materials get carried over?
================================================
For the mesh component, this script will attempt to create an atom mesh component
that uses the same material, pre-supposing that you have already run the 
LegacyAssetConverter script to generate Atom .material files out of legacy .mtl files.
(Gems\AtomLyIntegration\TechnicalArt\DccScriptingInterface\SDK\Maya\Scripts\Python\legacy_asset_converter\main.py)
For meshes that only have one sub-mesh, this is straightforward as the mesh will only
have one material to apply, and this script will look for a material with the same
name but a .material extension.

For mult-materials, this is a little tricky since Atom does not follow the same
ordered sub-material convention used by the legacy renderer. However, legacy .mtl
files that were generated when adding a .fbx to a project use a naming convention
that can be used by this script to match with the default materials that come from Atom.
So as long as you were using the initial .mtl generated by Lumberyard and have not
re-named the submaterials, it should find a match. This applies to both the
ActorComponent and the MeshComponent


What does this script needs?
================================================
This script will parse the assetcatalog.xml from the cache folder to get asset ids and path.
This file is in binary by default, you need to rebuild the engine with a modification to make it useable by this script.
    - In lumberyard, modify dev\Code\Tools\AssetProcessor\native\AssetManager\AssetCatalog.cpp line 343 to use AZ::ObjectStream::ST_XML instead of ST_BINARY
    - In o3de (if you use assetCatalogOverridePath), modify Code\Framework\AzFramework\AzFramework\Asset\AssetCatalog.cpp 
      and Code\Tools\AssetProcessor\native\AssetManager\AssetCatalog.cpp in the same fashion

Note that after the modification and rebuild, you will need to delete the cache folder and launch the engine for the catalog to be rebuilt

How do I run this script from a command line?
================================================
1) Check out any .slice, .layer, .ly, and .cry files you want to convert from source control
    - This script will remove the legacy components entirely, so make sure you have your files
      backed up before you run this script in case you want to run it again
2) From the Lumberyard root folder, run LegacyComponentConverter.py -project=<ProjectName> -include_gems -assetCatalogOverridePath=<ExternalProjectPath\Cache\pc\assetcatalog.xml>
    - -include_gems is optional. If you include Gems, it will run all all Gems, not just the ones enabled by your project
    - -assetCatalogOverridePath is optional. It will replace mesh/material asset id from the source project to this target project
      based on relative asset path from the project root (in general, Objects/Models/yourasset.fbx)


What are the artifacts of this script?
================================================
The .slice, .layer, .ly, and .cry files will be converted in-place. No new files will be created


Is this script destructive?
================================================
Yes! This is a one-way conversion that will clear the old data once converted. You should back up
your files before running this conversion in case you want to modify the script and re-run it on
your original level.
"""
CONVERTED_LOG_NAME = "ComponentConversion_ConvertedLegacyFiles.log"
UNCONVERTED_LOG_NAME = "ComponentConversion_UnsupportedLegacyFiles.log"
STATS_LOG_NAME = "ComponentConversion_LegacyComponentStats.log"

# Normal imports
import sys
import xml.etree.ElementTree as ET
import time
from zipfile import ZipFile
import tempfile
import subprocess

# Local python files
from LegacyConversionHelpers import *
from LegacyMeshComponentConverter import *
from LegacyMaterialComponentConverter import *
from LegacyActorComponentConverter import *
from LegacyPointLightComponentConverter import *
from LegacyTransformComponentConverter import *

BUILD_PATH = "./" # Use current working directory, we expect to be in lumberyard dev folder
GEMS_PATH = os.path.join(BUILD_PATH, "Gems")

class Component_File(object):
    """
    Class to perform any read, write or conversion operations on material (*.mtl) files.
    """
    def __init__(self, filename, projectDir, assetCatalogHelper, statsCollector):
        self.filename = filename
        self.normalizedProjectDir = os.path.normpath(projectDir)
        self.needsConversion = False
        self.hadException = False        
        self.assetCatalogHelper = assetCatalogHelper
        self.materialComponentConverter = Material_Component_Converter(assetCatalogHelper)#TODO - I'm pretty sure this is dead code
        self.xml = None
        self.statsCollector = statsCollector

        self.parse_xml()

    def is_valid_xml(self):
        """
        Performs a simple check to determine if the XML of the mtl is valid.
        This is to prevent an assert on a material conversion operation and
        preventing the script from finishing.

        It's possible for a material's xml to be malformed, which is why this check is needed.
        """
        return True
        try:
            if isinstance(self.xml.getroot(), xml.etree.ElementTree.Element):
                return True
            else:
                return False
        except:
            return False

    def parse_xml(self):
        """
        Open and parse the file's xml, storing it for access later.
        For .ly and .cry files, it will get the xml out of the .zip
        """
        if self.filename.endswith(".cry") or self.filename.endswith(".ly"):            
            zipRead = ZipFile(self.filename, 'r')
            
            contents = zipRead.read("levelentities.editor_xml")
            
            zipRead.close()

            # write the contents to a temporary file so we can parse it with ElementTree
            tmpFile = tempfile.NamedTemporaryFile(delete=False)
            tmpFile.write(contents)
            tmpFile.close()
            
            self.xml = xml.etree.ElementTree.parse(tmpFile.name)
            self.gather_elements()
            
            os.unlink(tmpFile.name)
            os.path.exists(tmpFile.name)
        elif os.path.exists(self.filename):
            #try:
                # TODO try-except is supposed to make it so one bad xml doesn't crash the lot
                # need to clean up stuff so the logging/conversion later doesn't crash
                # for now, better to crash here so we see where the exception is being thrown
            self.xml = xml.etree.ElementTree.parse(self.filename)
            self.gather_elements()                
            #except OSError as err:
            #    print("OS error: {0}".format(err))
            #    self.xml = None
            #    self.needsConversion = False
            #    self.hadException = True
            #except ValueError:
            #    print("Could not convert data to an integer.")
            #    self.xml = None
            #    self.needsConversion = False
            #    self.hadException = True
            #except:
            #    print("Unexpected error:", sys.exc_info()[0])
            #    self.xml = None
            #    self.needsConversion = False
            #    self.hadException = True

    def gather_elements(self):
        """
        Once the xml has been parsed, mine through it to find all of the
        neccessary elements that need to be modified.
        """
        print("starting to parse {0}".format(self.filename))

        componentConverters = []
        componentConverters.append(Mesh_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
        componentConverters.append(Actor_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
        componentConverters.append(Point_Light_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
        componentConverters.append(Transform_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))

        if self.is_valid_xml():
            root = self.xml.getroot()
            if root.tag == "ObjectStream" or True:
                # First, get a dictionary of child->parent mapping for later use inserting sibling elements                
                self.parent_map = {c:p for p in root.iter('Class') for c in p}

                # Now go through and look for mesh components
                for child in root.iter('Class'):
                    # If we run into one of the components we just added, skip it. It doesn't need to be converted,
                    # and it doesn't exist in the pre-built parent_map so it would throw an exception if we tried to access it
                    if child in self.parent_map:
                        parent = self.parent_map[child]
                        for componentConverter in componentConverters:
                            componentConverter.reset()
                            if componentConverter.is_this_the_component_im_looking_for(child, parent):
                                self.needsConversion = True
                                componentConverter.gather_info_for_conversion(child, parent)
                                # TODO - we're about to change the tree structure while iterating, which is apparently undefined but appears to work. Might be better to just build up a list of things that need to be modified, then do a second pass to replace the legacy component
                                # Seems to be okay since we only change or add elements, never remove entirely
                                componentConverter.convert(child, parent)
            self.xml._setroot(root)    
            # pretty print 
            ET.indent(self.xml, space='\t')

        print("finished parsing {0}".format(self.filename))

    def can_be_converted(self):
        """
        Determines if this material file can be converted by checking if
        it is using the Illum Shader
        """
        return self.needsConversion

    def can_write(self):
        """
        Checks to make sure the mtl file is writable.
        This is to prevent the script from asserting during a
        save attempt and preventing the script from finishing.
        """
        fullFilePath = self.get_atom_file_path()
        if os.path.exists(fullFilePath):
            if os.access(fullFilePath, os.W_OK):
                return True
        else:
            with open(fullFilePath,"a+") as f:
                f.close()
                return True
        return False

    def get_atom_file_path(self):
        # This is just a way to optionally create a new file for comparing with the original
        # TODO: control this via command line
        atomFileName = self.filename
        return atomFileName#.replace('.slice', '_atom.slice')

    def convert(self):
        """
        Creates the new level/slice file
        """
        # TODO - will not work if .slice is part of the path instead of the extension
        if self.needsConversion:
            if self.filename.endswith(".cry") or self.filename.endswith(".ly"):             
                # We can't just update the .cry file, we need to rebuild all the contents
                
                #Make a temporary file
                tmpFile, tmpFileName = tempfile.mkstemp(dir=os.path.dirname(self.filename))
                os.close(tmpFile)
                
                #Create a temporary copy of the .cry file 
                with ZipFile(self.filename, 'r') as zin:
                    with ZipFile(tmpFileName, 'w') as zout:
                        #Loop through the file list and write out every file but level.editor_xml with no modifications
                        #when we hit the level data we want to edit, write it out with the new contents
                        for item in zin.infolist():
                            if item.filename == "levelentities.editor_xml":
                                xmlString = xml.etree.ElementTree.tostring(self.xml.getroot())
                                zout.writestr(item, xmlString)
                            else:
                                zout.writestr(item, zin.read(item.filename))
                        
                #Remove old cry file and rename the temp file
                os.remove(self.filename)
                os.rename(tmpFileName, self.filename)
            else:
                self.xml.write(self.get_atom_file_path())

        return False
    
    def getUpdatedStatsCollector(self):
        """
        Returns the stats collector that was passed in intially, with any modifications that were made
        """
        return self.statsCollector

###############################################################################
def main():
    '''sys.__name__ wrapper function'''

    msgStr = "This tool will scan all of your lumberyard project's level/layer/slice files\n\
convert any compatible legacy components into the equivalent Atom components\n\
This script will overwrite the original files, and will remove the legacy components\n\
upon conversion, decimating the previous contents of those components.\n"

    commandLineOptions = Common_Command_Line_Options(sys.argv)
    if commandLineOptions.isHelp:
        print (commandLineOptions.helpString)
        return

    start_time = time.time()
    total_converted = 0

    extensionList = [".slice", ".layer", ".ly", ".cry"]
    fileList = get_file_list(commandLineOptions.projectName, commandLineOptions.includeGems, extensionList, BUILD_PATH, GEMS_PATH)
    
    if commandLineOptions.assetCatalogOverridePath:
        assetCatalogPath = commandLineOptions.assetCatalogOverridePath
        assetCatalogDictionaries = get_asset_catalog_dictionaries(assetCatalogPath)
    else:
        assetCatalogPath = os.path.join("Cache", commandLineOptions.projectName, get_default_asset_platform(), commandLineOptions.projectName, "assetcatalog.xml")
        assetCatalogDictionaries = get_asset_catalog_dictionaries(assetCatalogPath)
        
    # Create a log file to store converted component file filenames
    # and to check to see if the component file has already been converted.
    convertedLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, CONVERTED_LOG_NAME))

    # Create a log file to store component file filenames that need conversion
    # but cannot becuase they are read only.
    unconvertedLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, UNCONVERTED_LOG_NAME), include_previous = False)

    statsLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, STATS_LOG_NAME), include_previous = False)
    statsCollector = Stats_Collector()

    # Go through each component file to perform the conversion on it
    print("==============================")
    componentFileIndex = -1
    for componentFileInfo in fileList:
        componentFileIndex += 1
        componentFileName = componentFileInfo.filename
        copmonentFileProjectDir = componentFileInfo.normalizedProjectDir
        print(componentFileName)
        
        
        #if convertedLogFile.has_line(componentFileName.lstrip(BUILD_PATH)): # Use this to only convert files that haven't already been converted
        #    print("--> Previously converted, not doing")
        #    continue
        if commandLineOptions.endsWithStr == "" or componentFileName.lower().endswith(commandLineOptions.endsWithStr.lower()):
            componentFile = Component_File(componentFileName, copmonentFileProjectDir, assetCatalogDictionaries, statsCollector)
            if componentFile.can_be_converted():
                if commandLineOptions.useP4:
                    subprocess.check_call(['p4', 'edit', componentFileName])
                if componentFile.can_write():
                    componentFile.convert()
                    convertedLogFile.add_line_no_duplicates(componentFile.get_atom_file_path())
                    print("--> Converted")
                    total_converted += 1
                else:
                    unconvertedLogFile.add_line_no_duplicates("{0} - cannot access file (read-only)".format(componentFile.get_atom_file_path()))
                    print("--> Could not write to destination component file (read-only). Not converted.")
            else:
                print("--> did not need conversion.")
            statsCollector = componentFile.getUpdatedStatsCollector()
        print("\n")

    # Fill out the stats log
    statsLogFile.add_line("Mesh/Actor Components without a material overrride: {0}".format(statsCollector.noMaterialOverrideCount))
    statsLogFile.add_line("Mesh/Actor Components with a material overrride: {0}".format(statsCollector.materialOverrideCount))
    statsLogFile.add_line("Total Mesh/Actor Components: {0}".format(statsCollector.noMaterialOverrideCount + statsCollector.materialOverrideCount))

    # Finally, save the log files to disk
    convertedLogFile.save()
    unconvertedLogFile.save()
    statsLogFile.save()

    total_time = time.time() - start_time

    log_str = "You can view a list of converted component files in this log file:\n{0}\\{1}".format(BUILD_PATH, CONVERTED_LOG_NAME)
    unconverted_log_str = "You can view a list of component files that were not converted in this log file:\n{0}\\{1}".format(BUILD_PATH, UNCONVERTED_LOG_NAME)
    stats_log_str = "You can view a list of component files stats, such as feature and shader usage, in this log file:\n{0}\\{1}".format(BUILD_PATH, STATS_LOG_NAME)

    print("==============================\n")
    print("Conversion completed in {0} seconds.\n".format(total_time))
    print("Converted {0} component file(s)".format(total_converted))

    # Inform the user about the log files
    print("{0}".format(log_str))
    print("{0}".format(unconverted_log_str))
    print("{0}".format(stats_log_str))


if __name__ == '__main__':
    # GLOBAL NOTE:
    # - All python scripts should execute through a main() function.
    main()