/* * Copyright (c) Contributors to the Open 3D Engine Project. * For complete copyright and license terms please see the LICENSE at the root of this distribution. * * SPDX-License-Identifier: Apache-2.0 OR MIT * */ #pragma once #include #include #include #include #include #include #include namespace AtomSampleViewer { struct ImageComparisonToleranceLevel; namespace ScreenshotPaths { //! Returns the path to the screenshots capture folder. //! @resolvePath indicates whether to call ResolvePath() which will produce a full path, or keep the shorter asset folder path. AZStd::string GetScreenshotsFolder(bool resolvePath); //! Returns the path to the local baseline folder, which stores copies of screenshots previously taken on this machine. //! @resolvePath indicates whether to call ResolvePath() which will produce a full path, or keep the shorter asset folder path. AZStd::string GetLocalBaselineFolder(bool resolvePath); //! Returns the path to the official baseline folder, which stores copies of expected screenshots saved in source control. //! @resolvePath indicates whether to call ResolvePath() which will produce a full path, or keep the shorter asset folder path. AZStd::string GetOfficialBaselineFolder(bool resolvePath); //! Returns the path to the local baseline image that corresponds to @forScreenshotFile AZStd::string GetLocalBaseline(const AZStd::string& forScreenshotFile); //! Returns the path to the official baseline image that corresponds to @forScreenshotFile AZStd::string GetOfficialBaseline(const AZStd::string& forScreenshotFile); } //! Collects data about each script run by the ScriptManager. //! This includes counting errors, checking screenshots, and providing a final report dialog. class ScriptReporter { public: // currently set to track the ScriptReport index and the ScreenshotTestInfo index. using ReportIndex = AZStd::pair; static constexpr const char* TestResultsFolder = "TestResults"; static constexpr const char* UserFolder = "user"; //! Set the list of available tolerance levels, so the report can suggest an alternate level that matches the actual results. void SetAvailableToleranceLevels(const AZStd::vector& toleranceLevels); //! Clears all recorded data. void Reset(); //! Invalidates the final results when displaying a report to the user. This can be used to highlight //! local changes that were made, and remind the user that these results should not be considered official. //! Use an empty string to clear the invalidation. void SetInvalidationMessage(const AZStd::string& message); //! Indicates that a new script has started processing. //! Any subsequent errors will be included as part of this script's report. void PushScript(const AZStd::string& scriptAssetPath); //! Indicates that the current script has finished executing. //! Any subsequent errors will be included as part of the prior script's report. void PopScript(); //! Returns whether there are active processing scripts (i.e. more PushScript() calls than PopScript() calls) bool HasActiveScript() const; //! Indicates that a new screenshot is about to be captured. bool AddScreenshotTest(const AZStd::string& path); //! Check the latest screenshot using default thresholds. void CheckLatestScreenshot(const ImageComparisonToleranceLevel* comparisonPreset); //! Opens the script report dialog. //! This displays all the collected script reporting data, provides links to tools for analyzing data like //! viewing screenshot diffs. It can be left open during processing and will update in real-time. void OpenReportDialog(); void HideReportDialog(); //! Called every frame to update the ImGui dialog void TickImGui(); //! Returns true if there are any errors or asserts in the script report bool HasErrorsAssertsInReport() const; struct ScriptResultsSummary { uint32_t m_totalAsserts = 0; uint32_t m_totalErrors = 0; uint32_t m_totalWarnings = 0; uint32_t m_totalScreenshotsCount = 0; uint32_t m_totalScreenshotsFailed = 0; uint32_t m_totalScreenshotWarnings = 0; }; //! Displays the script results summary in ImGui. void DisplayScriptResultsSummary(); //! Retrieves the current script result summary. const ScriptResultsSummary& GetScriptResultSummary() const; struct ImageComparisonResult { enum class ResultCode { None, Pass, FileNotFound, FileNotLoaded, WrongSize, WrongFormat, NullImageComparisonToleranceLevel, ThresholdExceeded }; ResultCode m_resultCode = ResultCode::None; float m_standardDiffScore = 0.0f; float m_filteredDiffScore = 0.0f; //!< The diff score after filtering out visually imperceptible differences. float m_finalDiffScore = 0.0f; //! The diff score that was used for comparison. May be m_diffScore or m_filteredDiffScore. AZStd::string GetSummaryString() const; }; //! Records all the information about a screenshot comparison test. struct ScreenshotTestInfo { AZStd::string m_screenshotFilePath; AZStd::string m_officialBaselineScreenshotFilePath; //!< The path to the official baseline image that is checked into source control AZStd::string m_localBaselineScreenshotFilePath; //!< The path to a local baseline image that was established by the user ImageComparisonToleranceLevel m_toleranceLevel; //!< Tolerance for checking against the official baseline image ImageComparisonResult m_officialComparisonResult; //!< Result of comparing against the official baseline image, for reporting test failure ImageComparisonResult m_localComparisonResult; //!< Result of comparing against a local baseline, for reporting warnings }; //! Records all the information about a single test script. struct ScriptReport : public AZ::Debug::TraceMessageBus::Handler { ~ScriptReport() { AZ::Debug::TraceMessageBus::Handler::BusDisconnect(); } bool OnPreAssert(const char* /*fileName*/, int /*line*/, const char* /*func*/, [[maybe_unused]] const char* message) override { ++m_assertCount; return false; } bool OnPreError(const char* /*window*/, const char* /*fileName*/, int /*line*/, const char* /*func*/, const char* message) override { if (AZStd::string::npos == AzFramework::StringFunc::Find(message, "Screenshot check failed")) { ++m_generalErrorCount; } else { ++m_screenshotErrorCount; } return false; } bool OnPreWarning(const char* /*window*/, const char* /*fileName*/, int /*line*/, const char* /*func*/, const char* message) override { if (AZStd::string::npos == AzFramework::StringFunc::Find(message, "Screenshot does not match the local baseline")) { ++m_generalWarningCount; } else { ++m_screenshotWarningCount; } return false; } AZStd::string m_scriptAssetPath; uint32_t m_assertCount = 0; uint32_t m_generalErrorCount = 0; uint32_t m_screenshotErrorCount = 0; uint32_t m_generalWarningCount = 0; uint32_t m_screenshotWarningCount = 0; AZStd::vector m_screenshotTests; }; const AZStd::vector& GetScriptReport() const { return m_scriptReports; } // For exporting test results void ExportTestResults(); void ExportImageDiff(const char* filePath, const ScreenshotTestInfo& screenshotTest); AZStd::string ExportImageDiff(const ScriptReport& scriptReport, const ScreenshotTestInfo& screenshotTest); void SortScriptReports(); private: static const ImGuiTreeNodeFlags FlagDefaultOpen = ImGuiTreeNodeFlags_OpenOnArrow | ImGuiTreeNodeFlags_OpenOnDoubleClick | ImGuiTreeNodeFlags_DefaultOpen; static const ImGuiTreeNodeFlags FlagDefaultClosed = ImGuiTreeNodeFlags_OpenOnArrow | ImGuiTreeNodeFlags_OpenOnDoubleClick; // Reports a script error using standard formatting that matches ScriptManager enum class TraceLevel { Error, Warning }; // Controls which results are shown to the user // Must match static const char* DiplayOptions in .cpp file enum DisplayOption : int { AllResults, WarningsAndErrors, ErrorsOnly }; static void ReportScriptError(const AZStd::string& message); static void ReportScriptWarning(const AZStd::string& message); static void ReportScriptIssue(const AZStd::string& message, TraceLevel traceLevel); static void ReportScreenshotComparisonIssue(const AZStd::string& message, const AZStd::string& expectedImageFilePath, const AZStd::string& actualImageFilePath, TraceLevel traceLevel); // Loads image data from a .png file. // @param imageComparisonResult will be set to an error code if the function fails // @param path the path the .png file // @param buffer will be filled with the raw image data from the .png file // @param size will be set to the image size of the .png file // @param format will be set to the pixel format of the .png file // @return true if the file was loaded successfully static bool LoadPngData(ImageComparisonResult& imageComparisonResult, const AZStd::string& path, AZStd::vector& buffer, AZ::RHI::Size& size, AZ::RHI::Format& format, TraceLevel traceLevel); // Compares two image files and updates the ImageComparisonResult accordingly. // Returns false if an error prevented the comparison. static bool DiffImages(ImageComparisonResult& imageComparisonResult, const AZStd::string& expectedImageFilePath, const AZStd::string& actualImageFilePath, TraceLevel traceLevel); // Copies all captured screenshots to the local baseline folder. These can be used as an alternative to the central baseline for comparison. void UpdateAllLocalBaselineImages(); // Copies a single captured screenshot to the local baseline folder. This can be used as an alternative to the central baseline for comparison. bool UpdateLocalBaselineImage(ScreenshotTestInfo& screenshotTest, bool showResultDialog); // Copies a single captured screenshot to the official baseline source folder. bool UpdateSourceBaselineImage(ScreenshotTestInfo& screenshotTest, bool showResultDialog); // Clears comparison result to passing with no errors or warnings void ClearImageComparisonResult(ImageComparisonResult& comparisonResult); // Show a message box to let the user know the results of updating local baseline images void ShowUpdateLocalBaselineResult(int successCount, int failureCount); const ImageComparisonToleranceLevel* FindBestToleranceLevel(float diffScore, bool filterImperceptibleDiffs) const; void ShowReportDialog(); void ShowScreenshotTestInfoTreeNode(const AZStd::string& header, ScriptReport& scriptReport, ScreenshotTestInfo& screenshotResult); void ShowDiffButton(const char* buttonLabel, const AZStd::string& imagePathA, const AZStd::string& imagePathB); // Generates a path to the exported test results file. AZStd::string GenerateTimestamp() const; AZStd::string GenerateAndCreateExportedImageDiffPath(const ScriptReport& scriptReport, const ScreenshotTestInfo& screenshotTest) const; AZStd::string GenerateAndCreateExportedTestResultsPath() const; // Generates a diff between two images of the same size. void GenerateImageDiff(AZStd::span img1, AZStd::span img2, AZStd::vector& buffer); ScriptReport* GetCurrentScriptReport(); AZStd::string SeeConsole(uint32_t issueCount, const char* searchString); AZStd::string SeeBelow(uint32_t issueCount); void HighlightTextIf(bool shouldSet, ImVec4 color); void ResetTextHighlight(); void HighlightTextFailedOrWarning(bool isFailed, bool isWarning); struct HighlightColorSettings { ImVec4 m_highlightPassed; ImVec4 m_highlightFailed; ImVec4 m_highlightWarning; void UpdateColorSettings(); }; AZStd::multimap> m_descendingThresholdReports; bool m_showReportsSortedByThreshold = true; ImGuiMessageBox m_messageBox; AZStd::vector m_availableToleranceLevels; AZStd::string m_invalidationMessage; AZStd::vector m_scriptReports; //< Tracks errors for the current active script AZStd::vector m_currentScriptIndexStack; //< Tracks which of the scripts in m_scriptReports is currently active bool m_showReportDialog = false; bool m_colorHasBeenSet = false; DisplayOption m_displayOption = DisplayOption::AllResults; bool m_forceShowUpdateButtons = false; //< By default, the "Update" buttons are visible only for failed screenshots. This forces them to be visible. bool m_forceShowExportPngDiffButtons = false; //< By default, "Export Png Diff" buttons are visible only for failed screenshots. This forces them to be visible. AZStd::string m_officialBaselineSourceFolder; //< Used for updating official baseline screenshots AZStd::string m_exportedTestResultsPath = "Click the 'Export Test Results' button."; //< Path to exported test results file (if exported). AZStd::string m_uniqueTimestamp; HighlightColorSettings m_highlightSettings; ScriptResultsSummary m_resultsSummary; // Flags set and used by ShowReportDialog() bool m_showAll; bool m_showWarnings; }; } // namespace AtomSampleViewer