Add function to export screenshot diffs in ASV. (#325)

* Add function to export screenshot diffs in ASV.

Signed-off-by: hershey5045 <[email protected]>

* Small refactor to use image diff instead of image comparison when a difference is generated.

Signed-off-by: hershey5045 <[email protected]>

* Correct location of export png diff button.

Signed-off-by: hershey5045 <[email protected]>

* Refactor test results path generation.

Signed-off-by: hershey5045 <[email protected]>

* Switch array_view to span.

Signed-off-by: hershey5045 <[email protected]>

Gem/Code/Source/Automation/ScriptReporter.cpp

@@ -150,6 +150,7 @@ namespace AtomSampleViewer
         m_scriptReports.clear();
         m_currentScriptIndexStack.clear();
         m_invalidationMessage.clear();
+        m_uniqueTimestamp = GenerateTimestamp();     
     }
 
     void ScriptReporter::SetInvalidationMessage(const AZStd::string& message)
@@ -239,6 +240,13 @@ namespace AtomSampleViewer
         }
     }
 
+    AZStd::string ScriptReporter::GenerateTimestamp() const
+    {
+        const AZStd::chrono::system_clock::time_point now = AZStd::chrono::system_clock::now();
+        const float timeFloat = AZStd::chrono::duration<float>(now.time_since_epoch()).count();
+        return AZStd::string::format("%.4f", timeFloat);
+    }
+
     const ImageComparisonToleranceLevel* ScriptReporter::FindBestToleranceLevel(float diffScore, bool filterImperceptibleDiffs) const
     {
         float thresholdChecked = 0.0f;
@@ -454,6 +462,7 @@ namespace AtomSampleViewer
             m_displayOption = (DisplayOption)displayOption;
 
             ImGui::Checkbox("Force Show 'Update' Buttons", &m_forceShowUpdateButtons);
+            ImGui::Checkbox("Force Show 'Export Png Diff' Buttons", &m_forceShowExportPngDiffButtons);
 
             bool showWarnings = (m_displayOption == DisplayOption::AllResults) || (m_displayOption == DisplayOption::WarningsAndErrors);
             bool showAll = (m_displayOption == DisplayOption::AllResults);
@@ -619,6 +628,38 @@ namespace AtomSampleViewer
                                 ShowDiffButton("View Diff", screenshotResult.m_officialBaselineScreenshotFilePath, screenshotResult.m_screenshotFilePath);
                                 ImGui::PopID();
 
+                                if ((m_forceShowExportPngDiffButtons || screenshotResult.m_officialComparisonResult.m_resultCode == ImageComparisonResult::ResultCode::ThresholdExceeded) && ImGui::Button("Export Png Diff"))
+                                {
+                                    const auto projectPath = AZ::Utils::GetProjectPath();
+                                    AZStd::string imageDiffPath;
+                                    AZStd::string scriptFilenameWithouExtension;
+                                    AzFramework::StringFunc::Path::GetFileName(scriptReport.m_scriptAssetPath.c_str(), scriptFilenameWithouExtension);
+                                    AzFramework::StringFunc::Path::StripExtension(scriptFilenameWithouExtension);
+
+                                    AZStd::string screenshotFilenameWithouExtension;
+                                    AzFramework::StringFunc::Path::GetFileName(screenshotResult.m_screenshotFilePath.c_str(), screenshotFilenameWithouExtension);
+                                    AzFramework::StringFunc::Path::StripExtension(screenshotFilenameWithouExtension);
+
+                                    AZStd::string timestring = GenerateTimestamp();
+
+                                    AZStd::string imageDiffFilename = "imageDiff_" +
+                                        scriptFilenameWithouExtension + "_" +
+                                        screenshotFilenameWithouExtension + "_" +
+                                        m_uniqueTimestamp +
+                                        ".png";
+                                    AzFramework::StringFunc::Path::Join(projectPath.c_str(), UserFolder, imageDiffPath);
+                                    AzFramework::StringFunc::Path::Join(imageDiffPath.c_str(), TestResultsFolder, imageDiffPath);
+                                    AzFramework::StringFunc::Path::Join(imageDiffPath.c_str(), imageDiffFilename.c_str(), imageDiffPath);
+
+                                    AZStd::string imageDiffFolderPath;
+                                    AzFramework::StringFunc::Path::GetFolderPath(imageDiffPath.c_str(), imageDiffFolderPath);
+                                    auto io = AZ::IO::LocalFileIO::GetInstance();
+                                    io->CreatePath(imageDiffFolderPath.c_str());
+
+                                    ExportImageDiff(imageDiffPath.c_str(), screenshotResult);
+                                    m_messageBox.OpenPopupMessage("Image Diff Exported Successfully", AZStd::string::format("The image diff file was saved in %s", imageDiffPath.c_str()).c_str());
+                                }
+
                                 if ((!screenshotPassed || m_forceShowUpdateButtons) && ImGui::Button("Update##Official"))
                                 {
                                     if (screenshotResult.m_localComparisonResult.m_resultCode == ImageComparisonResult::ResultCode::FileNotFound)
@@ -1083,9 +1124,8 @@ namespace AtomSampleViewer
                 }
             }
         }
-
     }
-     
+
     void ScriptReporter::ExportTestResults()
     {
         m_exportedTestResultsPath = GenerateAndCreateExportedTestResultsPath();
@@ -1126,16 +1166,33 @@ namespace AtomSampleViewer
         }
     }
 
+    void ScriptReporter::ExportImageDiff(const char* filePath, const ScreenshotTestInfo& screenshotTestInfo)
+    {
+        using namespace AZ::Utils;
+        PngFile officialBaseline = PngFile::Load(screenshotTestInfo.m_officialBaselineScreenshotFilePath.c_str());
+        PngFile actualScreenshot = PngFile::Load(screenshotTestInfo.m_screenshotFilePath.c_str());
+
+        const size_t bufferSize = officialBaseline.GetBuffer().size();
+
+        AZStd::vector<uint8_t> diffBuffer = AZStd::vector<uint8_t>(bufferSize);
+        GenerateImageDiff(officialBaseline.GetBuffer(), actualScreenshot.GetBuffer(), diffBuffer);
+
+        AZStd::vector<uint8_t> buffer = AZStd::vector<uint8_t>(bufferSize * 3);
+        memcpy(buffer.data(), officialBaseline.GetBuffer().data(), bufferSize);
+        memcpy(buffer.data() + bufferSize, actualScreenshot.GetBuffer().data(), bufferSize);
+        memcpy(buffer.data() + bufferSize * 2, diffBuffer.data(), bufferSize);
+
+        PngFile imageDiff = PngFile::Create(RHI::Size(officialBaseline.GetWidth(), officialBaseline.GetHeight() * 3, 1), RHI::Format::R8G8B8A8_UNORM, buffer);
+        imageDiff.Save(filePath);
+    }
+
     AZStd::string ScriptReporter::GenerateAndCreateExportedTestResultsPath() const
     {
         // Setup our variables for the exported test results path and .txt file.
         const auto projectPath = AZ::Utils::GetProjectPath();
-        const AZStd::chrono::system_clock::time_point now = AZStd::chrono::system_clock::now();
-        const float timeFloat = AZStd::chrono::duration<float>(now.time_since_epoch()).count();
-        const AZStd::string timeString = AZStd::string::format("%.4f", timeFloat);
-        const AZStd::string exportFileName = AZStd::string::format("exportedTestResults_%s.txt", timeString.c_str());
+        const AZStd::string exportFileName = AZStd::string::format("exportedTestResults_%s.txt", m_uniqueTimestamp.c_str());
         AZStd::string exportTestResultsFolder;
-        AzFramework::StringFunc::Path::Join(projectPath.c_str(), "TestResults/", exportTestResultsFolder);
+        AzFramework::StringFunc::Path::Join(projectPath.c_str(), TestResultsFolder, exportTestResultsFolder);
 
         // Create the exported test results path & return .txt file path.
         auto io = AZ::IO::LocalFileIO::GetInstance();
@@ -1145,4 +1202,27 @@ namespace AtomSampleViewer
 
         return exportFile;
     }
+
+    void ScriptReporter::GenerateImageDiff(AZStd::span<const uint8_t> img1, AZStd::span<const uint8_t> img2, AZStd::vector<uint8_t>& buffer)
+    {
+        static constexpr size_t BytesPerPixel = 4;
+        static constexpr float MinDiffFilter = 0.01;
+        static constexpr uint8_t DefaultPixelValue = 122;
+
+        memset(buffer.data(), DefaultPixelValue, buffer.size() * sizeof(uint8_t));
+
+        for (size_t i = 0; i < img1.size(); i += BytesPerPixel)
+        {
+            const int16_t maxDiff = AZ::Utils::CalcMaxChannelDifference(img1, img2, i);
+
+            if (maxDiff / 255.0f > MinDiffFilter)
+            {
+                buffer[i] = aznumeric_cast<uint8_t>(maxDiff);
+                buffer[i + 1] = 0;
+                buffer[i + 2] = 0;
+            }
+            buffer[i + 3] = 255;
+        }
+    }
+
 } // namespace AtomSampleViewer

Gem/Code/Source/Automation/ScriptReporter.h

@@ -13,6 +13,7 @@
 #include <Atom/Utils/ImageComparison.h>
 #include <Automation/ImageComparisonConfig.h>
 #include <Utils/ImGuiMessageBox.h>
+#include <Atom/Utils/PngFile.h>
 
 namespace AtomSampleViewer
 {
@@ -44,6 +45,8 @@ namespace AtomSampleViewer
     class ScriptReporter
     {
     public:
+        static constexpr const char* TestResultsFolder = "TestResults";
+        static constexpr const char* UserFolder = "user";
 
         //! Set the list of available tolerance levels, so the report can suggest an alternate level that matches the actual results.
         void SetAvailableToleranceLevels(const AZStd::vector<ImageComparisonToleranceLevel>& toleranceLevels);
@@ -84,9 +87,6 @@ namespace AtomSampleViewer
         //! Returns true if there are any errors or asserts in the script report
         bool HasErrorsAssertsInReport() const;
 
-        // For exporting test results
-        void ExportTestResults();
-
         struct ImageComparisonResult
         {
             enum class ResultCode
@@ -177,6 +177,10 @@ namespace AtomSampleViewer
         
         const AZStd::vector<ScriptReport>& GetScriptReport() const { return m_scriptReports; }
 
+        // For exporting test results
+        void ExportTestResults();
+        void ExportImageDiff(const char* filePath, const ScreenshotTestInfo& screenshotTestInfo);
+
     private:
 
         // Reports a script error using standard formatting that matches ScriptManager
@@ -235,8 +239,12 @@ namespace AtomSampleViewer
         void ShowDiffButton(const char* buttonLabel, const AZStd::string& imagePathA, const AZStd::string& imagePathB);
 
         // Generates a path to the exported test results file.
+        AZStd::string GenerateTimestamp() const;
         AZStd::string GenerateAndCreateExportedTestResultsPath() const;
 
+        // Generates a diff between two images of the same size.
+        void GenerateImageDiff(AZStd::span<const uint8_t> img1, AZStd::span<const uint8_t> img2, AZStd::vector<uint8_t>& buffer);
+
         ScriptReport* GetCurrentScriptReport();
 
         ImGuiMessageBox m_messageBox;
@@ -249,8 +257,10 @@ namespace AtomSampleViewer
         bool m_showReportDialog = false;
         DisplayOption m_displayOption = DisplayOption::AllResults;
         bool m_forceShowUpdateButtons = false; //< By default, the "Update" buttons are visible only for failed screenshots. This forces them to be visible.
+        bool m_forceShowExportPngDiffButtons = false; //< By default, "Export Png Diff" buttons are visible only for failed screenshots. This forces them to be visible.
         AZStd::string m_officialBaselineSourceFolder; //< Used for updating official baseline screenshots
         AZStd::string m_exportedTestResultsPath = "Click the 'Export Test Results' button."; //< Path to exported test results file (if exported).
+        AZStd::string m_uniqueTimestamp;
     };
 
 } // namespace AtomSampleViewer