VisualTests: Add comparisons with previous captures, single comparison or for the whole test suite. Writes a diff image if they are not equal.

CMakeLists.txt

@@ -78,6 +78,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
 		option(BUILD_UNIT_TESTS "Enable to build UnitTests." ON)
 		option(BUILD_VISUAL_TESTS "Enable to build VisualTests." ON)
 		set(VISUAL_TESTS_DIRECTORIES "" CACHE STRING "Specify additional directories containing *.rml documents for the visual tests. Backslashes must be escaped. Separate multiple directories by semicolon.")
+		set(VISUAL_TESTS_INPUT_DIRECTORY "" CACHE PATH "Set the input directory for screenshot comparison performed by the VisualTests.")
 		set(VISUAL_TESTS_OUTPUT_DIRECTORY "" CACHE PATH "Set the output directory for screenshots generated by the VisualTests.")
 		option(BUILD_BENCHMARKS "Enable to build Benchmarks." ON)
 	endif()

Tests/CMakeLists.txt

@@ -72,6 +72,9 @@ if(BUILD_VISUAL_TESTS)
 	if(VISUAL_TESTS_DIRECTORIES)
 		target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_DIRECTORIES="${VISUAL_TESTS_DIRECTORIES}")
 	endif()
+	if(VISUAL_TESTS_INPUT_DIRECTORY)
+		target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_INPUT_DIRECTORY="${VISUAL_TESTS_INPUT_DIRECTORY}")
+	endif()
 	if(VISUAL_TESTS_OUTPUT_DIRECTORY)
 		target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY="${VISUAL_TESTS_OUTPUT_DIRECTORY}")
 	endif()

Tests/Output/.gitignore

@@ -1 +1,2 @@
 *.png
+*.log

Tests/Output/Readme.txt

@@ -1,3 +1,3 @@
-By default, the VisualTests suite will output screenshots into this directory.
+By default, the VisualTests suite will output screenshots and diff images into this directory, and read previous screenshots from this directory.
 
-Use the CMake option VISUAL_TESTS_OUTPUT_DIRECTORY to specify another directory.
+Use the CMake options VISUAL_TESTS_OUTPUT_DIRECTORY and VISUAL_TESTS_INPUT_DIRECTORY to specify other directories.

Tests/Source/VisualTests/CaptureScreen.cpp

@@ -0,0 +1,214 @@
+/*
+ * This source file is part of RmlUi, the HTML/CSS Interface Middleware
+ *
+ * For the latest information, see http://github.com/mikke89/RmlUi
+ *
+ * Copyright (c) 2008-2010 CodePoint Ltd, Shift Technology Ltd
+ * Copyright (c) 2019 The RmlUi Team, and contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+#include "CaptureScreen.h"
+#include <Shell.h>
+#include <ShellRenderInterfaceOpenGL.h>
+#include <RmlUi/Core/Log.h>
+#include <RmlUi/Core/StringUtilities.h>
+#include <cmath>
+
+#define LODEPNG_NO_COMPILE_CPP
+
+#include <lodepng.h>
+
+
+Rml::String GetInputDirectory()
+{
+#ifdef RMLUI_VISUAL_TESTS_INPUT_DIRECTORY
+	const Rml::String input_directory = Rml::String(RMLUI_VISUAL_TESTS_INPUT_DIRECTORY);
+#else
+	const Rml::String input_directory = Shell::FindSamplesRoot() + "../Tests/Output";
+#endif
+	return input_directory;
+}
+
+Rml::String GetOutputDirectory()
+{
+#ifdef RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY
+	const Rml::String output_directory = Rml::String(RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY);
+#else
+	const Rml::String output_directory = Shell::FindSamplesRoot() + "../Tests/Output";
+#endif
+	return output_directory;
+}
+
+
+bool CaptureScreenshot(ShellRenderInterfaceOpenGL* shell_renderer, const Rml::String& filename, int clip_width)
+{
+	using Image = ShellRenderInterfaceOpenGL::Image;
+
+	Image image_orig = shell_renderer->CaptureScreen();
+
+	if (!image_orig.data)
+	{
+		Rml::Log::Message(Rml::Log::LT_ERROR, "Could not capture screenshot from OpenGL window.");
+		return false;
+	}
+
+	if (clip_width == 0)
+		clip_width = image_orig.width;
+
+	// Create a new image flipped vertically, and clipped to the given clip width.
+	Image image;
+	image.width = clip_width;
+	image.height = image_orig.height;
+	image.num_components = image_orig.num_components;
+	image.data = Rml::UniquePtr<Rml::byte[]>(new Rml::byte[image.width * image.height * image.num_components]);
+
+	const int c = image.num_components;
+	for (int y = 0; y < image.height; y++)
+	{
+		const int flipped_y = image_orig.height - y - 1;
+
+		const int yb = y * image.width * c;
+		const int yb_orig = flipped_y * image_orig.width * c;
+		const int wb = image.width * c;
+
+		for (int xb = 0; xb < wb; xb++)
+		{
+			image.data[yb + xb] = image_orig.data[yb_orig + xb];
+		}
+	}
+
+	const Rml::String output_path = GetOutputDirectory() + "/" + filename;
+	unsigned int lodepng_result = lodepng_encode24_file(output_path.c_str(), image.data.get(), image.width, image.height);
+	if (lodepng_result)
+	{
+		Rml::Log::Message(Rml::Log::LT_ERROR, "Could not write the captured screenshot to %s: %s", output_path.c_str(), lodepng_error_text(lodepng_result));
+		return false;
+	}
+
+	return true;
+}
+
+
+struct DeferFree {
+	unsigned char* ptr = nullptr;
+	~DeferFree() { free(ptr); }
+};
+
+
+ComparisonResult CompareScreenToPreviousCapture(ShellRenderInterfaceOpenGL* shell_renderer, const Rml::String& filename)
+{
+	using Image = ShellRenderInterfaceOpenGL::Image;
+
+	const Rml::String input_path = GetInputDirectory() + "/" + filename;
+
+	unsigned char* data_ref = nullptr;
+	unsigned int w_ref = 0, h_ref = 0;
+
+	unsigned int lodepng_result = lodepng_decode24_file(&data_ref, &w_ref, &h_ref, input_path.c_str());
+	DeferFree defer_free{ data_ref };
+
+	if (lodepng_result)
+	{
+		ComparisonResult result{ false };
+		result.error_msg = Rml::CreateString(1024, "Could not read the captured screenshot from %s: %s", input_path.c_str(), lodepng_error_text(lodepng_result));
+		return result;
+	}
+	RMLUI_ASSERT(w_ref > 0 && h_ref > 0 && data_ref);
+	
+
+	Image screen = shell_renderer->CaptureScreen();
+	if (!screen.data)
+	{
+		ComparisonResult result{ false };
+		result.error_msg = "Could not capture screen from OpenGL window.";
+		return result;
+	}
+	RMLUI_ASSERT(screen.num_components == 3);
+
+	Image diff;
+	diff.width = w_ref;
+	diff.height = h_ref;
+	diff.num_components = 3;
+	diff.data = Rml::UniquePtr<Rml::byte[]>(new Rml::byte[diff.width * diff.height * diff.num_components]);
+
+	// So we have both images now, compare them! Also create a diff image.
+	// In case they are not the same size, we require that the reference image size is smaller or equal to the screen
+	// in both dimensions, and we compare them at the top-left corner.
+	// Note that the loaded image is flipped vertically compared to the OpenGL capture!
+
+	if (screen.width < (int)w_ref || screen.height < (int)h_ref)
+	{
+		ComparisonResult result{ false };
+		result.error_msg = "Test comparison failed. The screen is smaller than the reference image in one or both dimensions.";
+		return result;
+	}
+
+	size_t sum_diff = 0;
+
+	constexpr int c = 3;
+	for (int y = 0; y < (int)h_ref; y++)
+	{
+		const int y_flipped_screen = screen.height - y - 1;
+		const int yb_screen = y_flipped_screen * screen.width * c;
+
+		const int wb_ref = w_ref * c;
+		const int yb_ref = y * w_ref * c;
+
+		for (int xb = 0; xb < wb_ref; xb++)
+		{
+			const int i_ref = yb_ref + xb;
+			diff.data[i_ref] = (Rml::byte)std::abs((int)data_ref[i_ref] - (int)screen.data[yb_screen + xb]);
+			sum_diff += (size_t)diff.data[i_ref];
+		}
+	}
+
+	ComparisonResult result;
+	result.success = true;
+	result.is_equal = (sum_diff == 0);
+	result.absolute_difference_sum = sum_diff;
+
+	const size_t max_diff = size_t(c * 255) * size_t(w_ref) * size_t(h_ref);
+	result.similarity_score = (sum_diff == 0 ? 1.0 : 1.0 - std::log(double(sum_diff)) / std::log(double(max_diff)));
+
+	// Write the diff image to file if they are not equal.
+	if (!result.is_equal)
+	{
+		const Rml::String output_path = GetOutputDirectory() + "/diff-" + filename;
+		lodepng_result = lodepng_encode24_file(output_path.c_str(), diff.data.get(), diff.width, diff.height);
+		if (lodepng_result)
+		{
+			// We still report it as a success.
+			result.error_msg = "Could not write output diff image to " + output_path + Rml::String(": ") + lodepng_error_text(lodepng_result);
+		}
+	}
+
+	return result;
+}
+
+
+// Suppress warnings emitted by lodepng
+#if defined(RMLUI_PLATFORM_WIN32) && !defined(__MINGW32__)
+#pragma warning(disable : 4334)
+#pragma warning(disable : 4267)
+#endif
+
+#include <lodepng.cpp>

Tests/Source/VisualTests/Screenshot.h → Tests/Source/VisualTests/CaptureScreen.h

@@ -33,10 +33,21 @@
 
 class ShellRenderInterfaceOpenGL;
 
+struct ComparisonResult {
+	bool success = false;
+	bool is_equal = false;
+	double similarity_score = 0;
+	std::size_t absolute_difference_sum = 0;
+	Rml::String error_msg;
+};
 
-bool CaptureScreenshot(ShellRenderInterfaceOpenGL* shell_renderer, const Rml::String& filename, int clip_width);
 
+Rml::String GetInputDirectory();
 Rml::String GetOutputDirectory();
 
+bool CaptureScreenshot(ShellRenderInterfaceOpenGL* shell_renderer, const Rml::String& filename, int clip_width);
+
+ComparisonResult CompareScreenToPreviousCapture(ShellRenderInterfaceOpenGL* shell_renderer, const Rml::String& filename);
+
 
 #endif

Tests/Source/VisualTests/Screenshot.cpp

@@ -1,106 +0,0 @@
-/*
- * This source file is part of RmlUi, the HTML/CSS Interface Middleware
- *
- * For the latest information, see http://github.com/mikke89/RmlUi
- *
- * Copyright (c) 2008-2010 CodePoint Ltd, Shift Technology Ltd
- * Copyright (c) 2019 The RmlUi Team, and contributors
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- */
-
-#include "Screenshot.h"
-#include <Shell.h>
-#include <ShellRenderInterfaceOpenGL.h>
-#include <RmlUi/Core/Log.h>
-
-#define LODEPNG_NO_COMPILE_CPP
-
-#include <lodepng.h>
-
-
-bool CaptureScreenshot(ShellRenderInterfaceOpenGL* shell_renderer, const Rml::String& filename, int clip_width)
-{
-	using Image = ShellRenderInterfaceOpenGL::Image;
-
-	Image image_orig = shell_renderer->CaptureScreen();
-
-	if (!image_orig.data)
-	{
-		Rml::Log::Message(Rml::Log::LT_ERROR, "Could not capture screenshot from OpenGL window.");
-		return false;
-	}
-
-	if (clip_width == 0)
-		clip_width = image_orig.width;
-
-	// Create a new image flipped vertically, and clipped to the given clip width.
-	Image image;
-	image.width = clip_width;
-	image.height = image_orig.height;
-	image.num_components = image_orig.num_components;
-	image.data = Rml::UniquePtr<Rml::byte[]>(new Rml::byte[image.width * image.height * image.num_components]);
-
-	const int c = image.num_components;
-	for (int y = 0; y < image.height; y++)
-	{
-		const int flipped_y = image_orig.height - y - 1;
-
-		const int yb = y * image.width * c;
-		const int yb_orig = flipped_y * image_orig.width * c;
-		const int wb = image.width * c;
-
-		for (int xb = 0; xb < wb; xb++)
-		{
-			image.data[yb + xb] = image_orig.data[yb_orig + xb];
-		}
-	}
-
-	const Rml::String output_directory = GetOutputDirectory() + "/" + filename;
-
-	if (lodepng_encode24_file(output_directory.c_str(), image.data.get(), image.width, image.height))
-	{
-		Rml::Log::Message(Rml::Log::LT_ERROR, "Could not write the captured screenshot to %s", output_directory.c_str());
-		return false;
-	}
-
-	return true;
-}
-
-
-Rml::String GetOutputDirectory()
-{
-#ifdef RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY
-	const Rml::String output_directory = Rml::String(RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY);
-#else
-	const Rml::String output_directory = Shell::FindSamplesRoot() + "../Tests/Output";
-#endif
-	return output_directory;
-}
-
-
-
-// Suppress warnings emitted by lodepng
-#if defined(RMLUI_PLATFORM_WIN32) && !defined(__MINGW32__)
-#pragma warning(disable : 4334)
-#pragma warning(disable : 4267)
-#endif
-
-#include <lodepng.cpp>

Tests/Source/VisualTests/TestNavigator.cpp

@@ -28,15 +28,16 @@
 
 #include "TestNavigator.h"
 #include "TestSuite.h"
-#include "Screenshot.h"
+#include "CaptureScreen.h"
 #include "TestViewer.h"
 #include <RmlUi/Core/Context.h>
 #include <RmlUi/Core/Element.h>
 #include <Shell.h>
+#include <cstdio>
 
 // When capturing frames it seems we need to wait at least an extra frame for the newly submitted
 // render to be read back out. If we don't wait, we end up saving a screenshot of the previous test.
-constexpr int capture_wait_frame_count = 2;
+constexpr int iteration_wait_frame_count = 2;
 
 
 TestNavigator::TestNavigator(ShellRenderInterfaceOpenGL* shell_renderer, Rml::Context* context, TestViewer* viewer, TestSuiteList test_suites)
@@ -57,32 +58,45 @@ TestNavigator::~TestNavigator()
 
 void TestNavigator::Update()
 {
-	if (capture_index >= 0 && capture_wait_frames > 0)
+	if(iteration_state != IterationState::None)
 	{
-		capture_wait_frames -= 1;
-	}
-	else if (capture_index >= 0)
-	{
-		RMLUI_ASSERT(capture_index < CurrentSuite().GetNumTests());
-		capture_wait_frames += capture_wait_frame_count;
+		RMLUI_ASSERT(iteration_index >= 0);
 
-		if (!CaptureCurrentView())
+		// Capture test document screenshots iteratively every nth frame.
+		if (iteration_wait_frames > 0)
 		{
-			StopCaptureFullTestSuite();
-			return;
-		}
-
-		capture_index += 1;
-
-		TestSuite& suite = CurrentSuite();
-		if (capture_index < suite.GetNumTests())
-		{
-			suite.SetIndex(capture_index);
-			LoadActiveTest();
+			iteration_wait_frames -= 1;
 		}
 		else
 		{
-			StopCaptureFullTestSuite();
+			RMLUI_ASSERT(iteration_index < CurrentSuite().GetNumTests());
+			iteration_wait_frames = iteration_wait_frame_count;
+
+			if (iteration_state == IterationState::Capture)
+			{
+				if (!CaptureCurrentView())
+				{
+					StopTestSuiteIteration();
+					return;
+				}
+			}
+			else if (iteration_state == IterationState::Comparison)
+			{
+				comparison_results.push_back(CompareCurrentView());
+			}
+
+			iteration_index += 1;
+
+			TestSuite& suite = CurrentSuite();
+			if (iteration_index < suite.GetNumTests())
+			{
+				suite.SetIndex(iteration_index);
+				LoadActiveTest();
+			}
+			else
+			{
+				StopTestSuiteIteration();
+			}
 		}
 	}
 }
@@ -127,10 +141,40 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
 				LoadActiveTest();
 			}
 		}
+		else if (key_identifier == Rml::Input::KI_F5)
+		{
+			if (key_ctrl && key_shift)
+				StartTestSuiteIteration(IterationState::Comparison);
+			else
+			{
+				ComparisonResult result = CompareCurrentView();
+				if (result.success)
+				{
+					if (result.is_equal)
+					{
+						Rml::Log::Message(Rml::Log::LT_INFO, "%s compares EQUAL to the reference image.",
+							CurrentSuite().GetFilename().c_str());
+					}
+					else
+					{
+						Rml::Log::Message(Rml::Log::LT_INFO, "%s compares NOT EQUAL to the reference image. See diff image written to %s.",
+							CurrentSuite().GetFilename().c_str(), GetOutputDirectory().c_str());
+					}
+
+					if (!result.error_msg.empty())
+						Rml::Log::Message(Rml::Log::LT_ERROR, "%s", result.error_msg.c_str());
+				}
+				else
+				{
+					Rml::Log::Message(Rml::Log::LT_ERROR, "Comparison of %s failed.\n%s", CurrentSuite().GetFilename().c_str(), result.error_msg.c_str());
+				}
+
+			}
+		}
 		else if (key_identifier == Rml::Input::KI_F7)
 		{
 			if (key_ctrl && key_shift)
-				StartCaptureFullTestSuite();
+				StartTestSuiteIteration(IterationState::Capture);
 			else
 				CaptureCurrentView();
 		}
@@ -151,9 +195,9 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
 		}
 		else if (key_identifier == Rml::Input::KI_ESCAPE)
 		{
-			if (capture_index >= 0)
+			if (iteration_state != IterationState::None)
 			{
-				StopCaptureFullTestSuite();
+				StopTestSuiteIteration();
 			}
 			else if (source_state != SourceType::None)
 			{
@@ -214,7 +258,7 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
 			{
 				if (goto_index > 0)
 				{
-					if (CurrentSuite().SetIndex(goto_index))
+					if (CurrentSuite().SetIndex(goto_index - 1))
 					{
 						LoadActiveTest();
 					}
@@ -223,6 +267,10 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
 						viewer->SetGoToText(Rml::CreateString(64, "Go To out of bounds.", goto_index));
 					}
 				}
+				else
+				{
+					viewer->SetGoToText("");
+				}
 				goto_index = -1;
 			}
 		}
@@ -236,54 +284,171 @@ void TestNavigator::LoadActiveTest()
 	viewer->ShowSource(source_state);
 }
 
+static Rml::String ImageFilenameFromTest(const TestSuite& suite)
+{
+	const Rml::String& filename = suite.GetFilename();
+	return filename.substr(0, filename.rfind('.')) + ".png";
+}
+
+ComparisonResult TestNavigator::CompareCurrentView()
+{
+	const Rml::String filename = ImageFilenameFromTest(CurrentSuite());
+
+	ComparisonResult result = CompareScreenToPreviousCapture(shell_renderer, filename);
+
+	return result;
+}
+
+
 bool TestNavigator::CaptureCurrentView()
 {
-	Rml::String filename = CurrentSuite().GetFilename();
-	filename = filename.substr(0, filename.rfind('.')) + ".png";
+	const Rml::String filename = ImageFilenameFromTest(CurrentSuite());
 	
 	bool result = CaptureScreenshot(shell_renderer, filename, 1060);
 	
 	return result;
 }
 
-void TestNavigator::StopCaptureFullTestSuite()
+void TestNavigator::StartTestSuiteIteration(IterationState new_iteration_state)
 {
-	const Rml::String output_directory = GetOutputDirectory();
+	if (iteration_state != IterationState::None || new_iteration_state == IterationState::None)
+		return;
+
+	source_state = SourceType::None;
+
+	if (new_iteration_state == IterationState::Comparison)
+		comparison_results.clear();
+
+	TestSuite& suite = CurrentSuite();
+	iteration_initial_index = suite.GetIndex();
+	iteration_wait_frames = iteration_wait_frame_count;
+
+	iteration_state = new_iteration_state;
+	iteration_index = 0;
+	suite.SetIndex(iteration_index);
+	LoadActiveTest();
+}
+
+static bool SaveFile(const Rml::String& file_path, const Rml::String& contents)
+{
+	std::FILE* file = std::fopen(file_path.c_str(), "wt");
+	if (!file)
+		return false;
+
+	std::fputs(contents.c_str(), file);
+	std::fclose(file);
+
+	return true;
+}
+
+void TestNavigator::StopTestSuiteIteration()
+{
+	if (iteration_state == IterationState::None)
+		return;
 
+	const Rml::String output_directory = GetOutputDirectory();
 	TestSuite& suite = CurrentSuite();
 	const int num_tests = suite.GetNumTests();
 
-	if (capture_index == num_tests)
+	if (iteration_state == IterationState::Capture)
 	{
-		Rml::Log::Message(Rml::Log::LT_INFO, "Successfully captured %d document screenshots to directory: %s", capture_index, output_directory.c_str());
+		if (iteration_index == num_tests)
+		{
+			Rml::Log::Message(Rml::Log::LT_INFO, "Successfully captured %d document screenshots to directory: %s", iteration_index, output_directory.c_str());
+		}
+		else
+		{
+			Rml::Log::Message(Rml::Log::LT_ERROR, "Test suite capture aborted after %d of %d test(s). Output directory: %s", iteration_index, num_tests, output_directory.c_str());
+		}
 	}
-	else
+	else if (iteration_state == IterationState::Comparison)
 	{
-		Rml::Log::Message(Rml::Log::LT_ERROR, "Test suite capture aborted after %d of %d test(s). Output directory: %s", capture_index, num_tests, output_directory.c_str());
-	}
+		RMLUI_ASSERT(iteration_index == (int)comparison_results.size());
 
-	suite.SetIndex(capture_initial_index);
-	LoadActiveTest();
+		// Indices
+		Rml::Vector<int> equal;
+		Rml::Vector<int> not_equal;
+		Rml::Vector<int> failed;
+		Rml::Vector<int> skipped;
+		 
+		for(int i = 0; i < (int)comparison_results.size(); i++)
+		{
+			const ComparisonResult& comparison = comparison_results[i];
 
-	capture_index = -1;
-	capture_initial_index = -1;
-	capture_wait_frames = -1;
-}
+			if (!comparison.success)
+				failed.push_back(i);
+			else if (comparison.is_equal)
+				equal.push_back(i);
+			else
+				not_equal.push_back(i);
+		}
+		for (int i = (int)comparison_results.size(); i < num_tests; i++)
+			skipped.push_back(i);
 
-void TestNavigator::StartCaptureFullTestSuite()
-{
-	if(capture_index == -1)
-	{
-		source_state = SourceType::None;
-
-		TestSuite& suite = CurrentSuite();
-		capture_initial_index = suite.GetIndex();
-		capture_wait_frames = capture_wait_frame_count;
-		
-		capture_index = 0;
-		suite.SetIndex(capture_index);
-		LoadActiveTest();
+		const Rml::String summary = Rml::CreateString(256, "  Total tests: %d\n  Equal: %d\n  Not equal: %d\n  Failed: %d\n  Skipped: %d",
+			num_tests, (int)equal.size(), (int)not_equal.size(), (int)failed.size(), (int)skipped.size());
+
+		if (iteration_index == num_tests)
+		{
+			Rml::Log::Message(Rml::Log::LT_INFO, "Compared all test documents to their screenshot captures.\n%s", summary.c_str());
+		}
+		else
+		{
+			Rml::Log::Message(Rml::Log::LT_ERROR, "Test suite comparison aborted after %d of %d test(s).\n%s", iteration_index, num_tests, summary.c_str());
+		}
+
+		Rml::String log;
+		log.reserve(comparison_results.size() * 100);
+
+		log += "RmlUi VisualTests comparison log output.\n---------------------------------------\n\n" + summary;
+		log += "\n\nEqual:\n";
+
+		for (int i : equal)
+		{
+			suite.SetIndex(i);
+			log += Rml::CreateString(256, "%5d   %s\n", i + 1, suite.GetFilename().c_str());
+		}
+		log += "\nNot Equal:\n";
+		if (!not_equal.empty())
+			log += "Percentages are similarity scores. Difference images written to " + GetOutputDirectory() + "/diff-*.png\n\n";
+		for (int i : not_equal)
+		{
+			suite.SetIndex(i);
+			log += Rml::CreateString(256, "%5d   %5.1f%%   %s\n", i + 1, comparison_results[i].similarity_score*100.0, suite.GetFilename().c_str());
+			if (!comparison_results[i].error_msg.empty())
+				log += "          " + comparison_results[i].error_msg + "\n";
+		}
+		log += "\nFailed:\n";
+		for (int i : failed)
+		{
+			suite.SetIndex(i);
+			log += Rml::CreateString(512, "%5d   %s\n", i + 1, suite.GetFilename().c_str());
+			log += "          " + comparison_results[i].error_msg + "\n";
+		}
+		log += "\nSkipped:\n";
+		for (int i : skipped)
+		{
+			suite.SetIndex(i);
+			log += Rml::CreateString(256, "%5d   %s\n", i + 1, suite.GetFilename().c_str());
+		}
+
+		const Rml::String log_path = GetOutputDirectory() + "/comparison.log";
+		bool save_result = SaveFile(log_path, log);
+		if (save_result && failed.empty())
+			Rml::Log::Message(Rml::Log::LT_INFO, "Comparison log output written to %s", log_path.c_str());
+		else if(save_result && !failed.empty())
+			Rml::Log::Message(Rml::Log::LT_ERROR, "Comparison log output written to %s.\nSome captures failed, see log output for details.", log_path.c_str());
+		else
+			Rml::Log::Message(Rml::Log::LT_ERROR, "Failed writing comparison log output to file %s", log_path.c_str());
 	}
+
+	suite.SetIndex(iteration_initial_index);
+	LoadActiveTest();
+
+	iteration_index = -1;
+	iteration_initial_index = -1;
+	iteration_wait_frames = -1;
+	iteration_state = IterationState::None;
 }
 
 

Tests/Source/VisualTests/TestNavigator.h

@@ -30,12 +30,13 @@
 #define RMLUI_TESTS_VISUALTESTS_NAVIGATION_H
 
 #include "TestSuite.h"
+#include "CaptureScreen.h"
 #include "TestViewer.h"
 #include <RmlUi/Core/Types.h>
 #include <RmlUi/Core/EventListener.h>
 
 class ShellRenderInterfaceOpenGL;
-
+struct ComparisonResult;
 
 class TestNavigator : public Rml::EventListener {
 public:
@@ -48,15 +49,18 @@ protected:
 	void ProcessEvent(Rml::Event& event) override;
 
 private:
+	enum class IterationState { None, Capture, Comparison };
+
 	TestSuite& CurrentSuite() { return test_suites[index]; }
 
 	void LoadActiveTest();
 
-	bool CaptureCurrentView();
+	ComparisonResult CompareCurrentView();
 
+	bool CaptureCurrentView();
 
-	void StartCaptureFullTestSuite();
-	void StopCaptureFullTestSuite();
+	void StartTestSuiteIteration(IterationState iteration_state);
+	void StopTestSuiteIteration();
 
 	ShellRenderInterfaceOpenGL* shell_renderer;
 	Rml::Context* context;
@@ -66,9 +70,14 @@ private:
 	int index = 0;
 	int goto_index = -1;
 	SourceType source_state = SourceType::None;
-	int capture_index = -1;
-	int capture_initial_index = -1;
-	int capture_wait_frames = -1;
+
+	IterationState iteration_state = IterationState::None;
+
+	int iteration_index = -1;
+	int iteration_initial_index = -1;
+	int iteration_wait_frames = -1;
+
+	Rml::Vector<ComparisonResult> comparison_results;
 };
 
 

Tests/Source/VisualTests/main.cpp

@@ -28,7 +28,7 @@
 
 #include "TestViewer.h"
 #include "TestNavigator.h"
-#include "Screenshot.h"
+#include "CaptureScreen.h"
 #include "TestSuite.h"
 #include <RmlUi/Core/Context.h>
 #include <RmlUi/Core/Core.h>