Browse Source

Improve visual tests suite.

Michael Ragazzon 5 years ago
parent
commit
a2101aa032

+ 6 - 4
Tests/Data/visual_tests_help.rml

@@ -19,7 +19,7 @@
 		font-size: 17px;
 		font-size: 17px;
 		width: 700px;
 		width: 700px;
 		margin: 0 auto;
 		margin: 0 auto;
-		padding-top: 50px;
+		padding-top: 30px;
 	}
 	}
 	p {
 	p {
 		clear: both;
 		clear: both;
@@ -34,14 +34,15 @@
 	key, value {
 	key, value {
 		display: block;
 		display: block;
 		line-height: 1.5;
 		line-height: 1.5;
-		width: 46%;
 	}
 	}
 	key {
 	key {
+		width: 40%;
 		float: left;
 		float: left;
 		clear: both;
 		clear: both;
 		text-align: right;
 		text-align: right;
 	}
 	}
 	value {
 	value {
+		width: 52%;
 		float: right;
 		float: right;
 		clear: right;
 		clear: right;
 	}
 	}
@@ -60,12 +61,13 @@
 <key>F8</key> <value>Open debugger</value>
 <key>F8</key> <value>Open debugger</value>
 <key>Ctrl+G</key> <value>Go To test (followed by number, enter)</value>
 <key>Ctrl+G</key> <value>Go To test (followed by number, enter)</value>
 <key>Ctrl+F</key> <value>Filter tests</value>
 <key>Ctrl+F</key> <value>Filter tests</value>
+<key>Ctrl+R</key> <value>Reload test</value>
 <key>Ctrl+S</key> <value>View source of test</value>
 <key>Ctrl+S</key> <value>View source of test</value>
 <key>Ctrl+Shift+S</key> <value>View source of reference</value>
 <key>Ctrl+Shift+S</key> <value>View source of reference</value>
-<key>Left Mouse Button on Link</key> <value>Copy link to clipboard</value>
+<key>Left Mouse Button on Link</key> <value>Copy link to clipboard (on supported platforms)</value>
 <key>F1</key> <value>Show / hide help</value>
 <key>F1</key> <value>Show / hide help</value>
 
 
-<p>Make sure to check the log output after comparisons and captures.</p>
+<p>Results of comparisons and captures are written to the log output.</p>
 
 
 <h1>Directories</h1>
 <h1>Directories</h1>
 <key>Test suite(s)</key> <span id="test_directories"><value>C:\</value> <value>E:\</value> <value>D:\</value></span>
 <key>Test suite(s)</key> <span id="test_directories"><value>C:\</value> <value>E:\</value> <value>D:\</value></span>

+ 11 - 2
Tests/Source/VisualTests/CaptureScreen.cpp

@@ -159,12 +159,15 @@ ComparisonResult CompareScreenToPreviousCapture(ShellRenderInterfaceOpenGL* shel
 		for (int xb = 0; xb < wb_ref; xb++)
 		for (int xb = 0; xb < wb_ref; xb++)
 		{
 		{
 			const int i_ref = yb_ref + xb;
 			const int i_ref = yb_ref + xb;
-			diff.data[i_ref] = (Rml::byte)std::abs((int)data_ref[i_ref] - (int)screen.data[yb_screen + xb]);
+			Rml::byte pix_ref = data_ref[i_ref];
+			Rml::byte pix_screen = screen.data[yb_screen + xb];
+			diff.data[i_ref] = (Rml::byte)std::abs((int)pix_ref - (int)pix_screen);
 			sum_diff += (size_t)diff.data[i_ref];
 			sum_diff += (size_t)diff.data[i_ref];
 		}
 		}
 	}
 	}
 
 
 	ComparisonResult result;
 	ComparisonResult result;
+	result.skipped = false;
 	result.success = true;
 	result.success = true;
 	result.is_equal = (sum_diff == 0);
 	result.is_equal = (sum_diff == 0);
 	result.absolute_difference_sum = sum_diff;
 	result.absolute_difference_sum = sum_diff;
@@ -175,7 +178,13 @@ ComparisonResult CompareScreenToPreviousCapture(ShellRenderInterfaceOpenGL* shel
 	// Write the diff image to file if they are not equal.
 	// Write the diff image to file if they are not equal.
 	if (!result.is_equal)
 	if (!result.is_equal)
 	{
 	{
-		const Rml::String output_path = GetCaptureOutputDirectory() + "/diff-" + filename;
+		Rml::String out_filename = filename;
+		size_t offset = filename.rfind('.');
+		if (offset == Rml::String::npos)
+			offset = filename.size();
+		out_filename.insert(offset, "-diff");
+
+		const Rml::String output_path = GetCaptureOutputDirectory() + '/' + out_filename;
 		lodepng_result = lodepng_encode24_file(output_path.c_str(), diff.data.get(), diff.width, diff.height);
 		lodepng_result = lodepng_encode24_file(output_path.c_str(), diff.data.get(), diff.width, diff.height);
 		if (lodepng_result)
 		if (lodepng_result)
 		{
 		{

+ 1 - 0
Tests/Source/VisualTests/CaptureScreen.h

@@ -34,6 +34,7 @@
 class ShellRenderInterfaceOpenGL;
 class ShellRenderInterfaceOpenGL;
 
 
 struct ComparisonResult {
 struct ComparisonResult {
+	bool skipped = true;
 	bool success = false;
 	bool success = false;
 	bool is_equal = false;
 	bool is_equal = false;
 	double similarity_score = 0;
 	double similarity_score = 0;

+ 25 - 13
Tests/Source/VisualTests/TestNavigator.cpp

@@ -87,7 +87,9 @@ void TestNavigator::Update()
 			}
 			}
 			else if (iteration_state == IterationState::Comparison)
 			else if (iteration_state == IterationState::Comparison)
 			{
 			{
-				comparison_results.push_back(CompareCurrentView());
+				RMLUI_ASSERT((int)comparison_results.size() == CurrentSuite().GetNumTests());
+				int test_index = CurrentSuite().GetIndex();
+				comparison_results[test_index] = CompareCurrentView();
 			}
 			}
 
 
 			iteration_index += 1;
 			iteration_index += 1;
@@ -156,6 +158,12 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
 		else if (key_identifier == Rml::Input::KI_F && key_ctrl)
 		else if (key_identifier == Rml::Input::KI_F && key_ctrl)
 		{
 		{
 			element_filter_input->Focus();
 			element_filter_input->Focus();
+			context->ProcessKeyDown(Rml::Input::KI_A, Rml::Input::KeyModifier::KM_CTRL);
+			context->ProcessKeyUp(Rml::Input::KI_A, Rml::Input::KeyModifier::KM_CTRL);
+		}
+		else if (key_identifier == Rml::Input::KI_R && key_ctrl)
+		{
+			LoadActiveTest();
 		}
 		}
 		else if (key_identifier == Rml::Input::KI_S && key_ctrl)
 		else if (key_identifier == Rml::Input::KI_S && key_ctrl)
 		{
 		{
@@ -354,10 +362,14 @@ void TestNavigator::StartTestSuiteIteration(IterationState new_iteration_state)
 
 
 	source_state = SourceType::None;
 	source_state = SourceType::None;
 
 
+	TestSuite& suite = CurrentSuite();
+
 	if (new_iteration_state == IterationState::Comparison)
 	if (new_iteration_state == IterationState::Comparison)
+	{
 		comparison_results.clear();
 		comparison_results.clear();
+		comparison_results.resize(suite.GetNumTests());
+	}
 
 
-	TestSuite& suite = CurrentSuite();
 	iteration_initial_index = suite.GetIndex();
 	iteration_initial_index = suite.GetIndex();
 	iteration_wait_frames = iteration_wait_frame_count;
 	iteration_wait_frames = iteration_wait_frame_count;
 
 
@@ -406,7 +418,7 @@ void TestNavigator::StopTestSuiteIteration()
 	}
 	}
 	else if (iteration_state == IterationState::Comparison)
 	else if (iteration_state == IterationState::Comparison)
 	{
 	{
-		RMLUI_ASSERT(iteration_index == (int)comparison_results.size());
+		RMLUI_ASSERT(num_tests == (int)comparison_results.size());
 
 
 		// Indices
 		// Indices
 		Rml::Vector<int> equal;
 		Rml::Vector<int> equal;
@@ -418,15 +430,15 @@ void TestNavigator::StopTestSuiteIteration()
 		{
 		{
 			const ComparisonResult& comparison = comparison_results[i];
 			const ComparisonResult& comparison = comparison_results[i];
 
 
-			if (!comparison.success)
+			if (comparison.skipped)
+				skipped.push_back(i);
+			else if (!comparison.success)
 				failed.push_back(i);
 				failed.push_back(i);
 			else if (comparison.is_equal)
 			else if (comparison.is_equal)
 				equal.push_back(i);
 				equal.push_back(i);
 			else
 			else
 				not_equal.push_back(i);
 				not_equal.push_back(i);
 		}
 		}
-		for (int i = (int)comparison_results.size(); i < num_tests; i++)
-			skipped.push_back(i);
 
 
 		Rml::String summary = Rml::CreateString(256, "  Total tests: %d\n  Equal: %d\n  Not equal: %d\n  Failed: %d\n  Skipped: %d",
 		Rml::String summary = Rml::CreateString(256, "  Total tests: %d\n  Equal: %d\n  Not equal: %d\n  Failed: %d\n  Skipped: %d",
 			num_tests, (int)equal.size(), (int)not_equal.size(), (int)failed.size(), (int)skipped.size());
 			num_tests, (int)equal.size(), (int)not_equal.size(), (int)failed.size(), (int)skipped.size());
@@ -451,14 +463,8 @@ void TestNavigator::StopTestSuiteIteration()
 		log.reserve(comparison_results.size() * 100);
 		log.reserve(comparison_results.size() * 100);
 
 
 		log += "RmlUi VisualTests comparison log output\n---------------------------------------\n\n" + summary;
 		log += "RmlUi VisualTests comparison log output\n---------------------------------------\n\n" + summary;
-		log += "\n\nEqual:\n";
+		log += "\n\nNot Equal:\n";
 
 
-		for (int i : equal)
-		{
-			suite.SetIndex(i);
-			log += Rml::CreateString(256, "%5d   %s\n", i + 1, suite.GetFilename().c_str());
-		}
-		log += "\nNot Equal:\n";
 		if (!not_equal.empty())
 		if (!not_equal.empty())
 			log += "Percentages are similarity scores. Difference images written to " + GetCaptureOutputDirectory() + "/diff-*.png\n\n";
 			log += "Percentages are similarity scores. Difference images written to " + GetCaptureOutputDirectory() + "/diff-*.png\n\n";
 		for (int i : not_equal)
 		for (int i : not_equal)
@@ -468,6 +474,12 @@ void TestNavigator::StopTestSuiteIteration()
 			if (!comparison_results[i].error_msg.empty())
 			if (!comparison_results[i].error_msg.empty())
 				log += "          " + comparison_results[i].error_msg + "\n";
 				log += "          " + comparison_results[i].error_msg + "\n";
 		}
 		}
+		log += "\nEqual:\n";
+		for (int i : equal)
+		{
+			suite.SetIndex(i);
+			log += Rml::CreateString(256, "%5d   %s\n", i + 1, suite.GetFilename().c_str());
+		}
 		log += "\nFailed:\n";
 		log += "\nFailed:\n";
 		for (int i : failed)
 		for (int i : failed)
 		{
 		{

+ 4 - 1
Tests/Tools/convert_css_test_suite_to_rml.py

@@ -76,7 +76,7 @@ def border_format(side: str, type: str, content: str):
 		return "border" + side + "-width: " + content
 		return "border" + side + "-width: " + content
 
 
 	# Next are the shorthand properties, they should contain max a single size, a single style, and a single color.
 	# Next are the shorthand properties, they should contain max a single size, a single style, and a single color.
-	width = re.search(r'\b([0-9]+[a-z]+)\b', content, flags = re.IGNORECASE)
+	width = re.search(r'\b([0-9]+(\.[0-9]+)?[a-z]+|0)\b', content, flags = re.IGNORECASE)
 	if width:
 	if width:
 		width = width.group(1)
 		width = width.group(1)
 
 
@@ -123,6 +123,9 @@ assert( border_find_replace("margin:10px; border:20px solid black; padding:30px;
 assert( border_find_replace(" border-left: 7px solid navy; border-right: 17px solid navy; } ") == ' border-left: 7px navy; border-right: 17px navy; } ' )
 assert( border_find_replace(" border-left: 7px solid navy; border-right: 17px solid navy; } ") == ' border-left: 7px navy; border-right: 17px navy; } ' )
 assert( border_find_replace(" border: blue solid 3px; ") == ' border: 3px blue; ' )
 assert( border_find_replace(" border: blue solid 3px; ") == ' border: 3px blue; ' )
 assert( border_find_replace(" border: solid lime; ") == ' border: 3px lime; ' )
 assert( border_find_replace(" border: solid lime; ") == ' border: 3px lime; ' )
+assert( border_find_replace(" border: 0; ") == ' border: 0 black; ' )
+assert( border_find_replace(" border-bottom: 0.25em solid green; ") == ' border-bottom: 0.25em green; ' )
+assert( border_find_replace(" border-width: 0; ") == ' border-width:  0; ' )
 assert( border_find_replace(" border-left: orange solid 1em; ") == ' border-left: 1em orange; ' )
 assert( border_find_replace(" border-left: orange solid 1em; ") == ' border-left: 1em orange; ' )
 assert( border_find_replace(" border-style: solid none solid solid; ") == ' border-width:  3px 0px 3px 3px; ' )
 assert( border_find_replace(" border-style: solid none solid solid; ") == ' border-width:  3px 0px 3px 3px; ' )
 assert( border_find_replace("   border: solid; border-style: solid none solid solid; border-style: solid solid solid none; ") == '   border: 3px black; border-width:  3px 0px 3px 3px; border-width:  3px 3px 3px 0px; ' )
 assert( border_find_replace("   border: solid; border-style: solid none solid solid; border-style: solid solid solid none; ") == '   border: 3px black; border-width:  3px 0px 3px 3px; border-width:  3px 3px 3px 0px; ' )