RML support for numeric character reference (Unicode escape sequences) (#401)

Source/Core/ElementText.cpp

@@ -424,7 +424,7 @@ void ElementText::OnPropertyChange(const PropertyIdSet& changed_properties)
 // Returns the RML of this element
 void ElementText::GetRML(String& content)
 {
-	content += text;
+	content += StringUtilities::EncodeRml(text);
 }
 
 // Updates the configuration this element uses for its font.

Source/Core/Factory.cpp

@@ -386,7 +386,7 @@ bool Factory::InstanceElementText(Element* parent, const String& in_text)
 	const bool only_white_space = std::all_of(text.begin(), text.end(), &StringUtilities::IsWhitespace);
 	if (only_white_space)
 		return true;
-
+	
 	// See if we need to parse it as RML, and whether the text contains data expressions (curly brackets).
 	bool parse_as_rml = false;
 	bool has_data_expression = false;
@@ -453,6 +453,9 @@ bool Factory::InstanceElementText(Element* parent, const String& in_text)
 			return false;
 		}
 
+		// Unescape any escaped entities or unicode symbols
+		text = StringUtilities::DecodeRml(text);
+	
 		text_element->SetText(text);
 
 		// Add to active node.

Source/Core/StringUtilities.cpp

@@ -32,6 +32,7 @@
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
+#include <sstream>
 
 namespace Rml {
 
@@ -152,6 +153,58 @@ String StringUtilities::DecodeRml(const String& s)
 				i += 6;
 				continue;
 			}
+			else if (s[i+1] == '#')
+			{
+				size_t start = i + 2;
+				if (s[i+2] == 'x')
+				{
+					start++;
+					size_t j = 0;
+					for(; j < 8; j++)
+					{
+						auto const& c = s[start + j];
+						if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')))
+							break;
+					}
+					
+					if (j > 0 && s[start + j] == ';')
+					{
+						String tmp = s.substr(start, j);
+						std::istringstream iss(tmp);
+						uint32_t code_point;
+						if (iss >> std::hex >> code_point)
+						{
+							result += ToUTF8(static_cast<Character>(code_point));
+							i = start + j + 1;
+							continue;
+						}
+					}
+				}
+				else
+				{
+					size_t j = 0;
+					for(; j < 8; j++)
+					{
+						auto const& c = s[start + j];
+						if (!(c >= '0' && c <= '9'))
+							break;
+					}
+					
+					if (j > 0 && s[start + j] == ';')
+					{
+						String tmp = s.substr(start, j);
+						std::istringstream iss(tmp);
+						uint32_t code_point;
+						if (iss >> code_point)
+						{
+							result += ToUTF8(static_cast<Character>(code_point));
+							i = start + j + 1;
+							continue;
+						}
+					}
+				}
+				
+			}
 		}
 		result += s[i];
 		i += 1;

Tests/Source/Benchmarks/Element.cpp

@@ -77,20 +77,7 @@ static int GetNumDescendentElements(Element* element)
 	return result;
 }
 
-static String GenerateRml(const int num_rows)
-{
-	static nanobench::Rng rng;
-
-	Rml::String rml;
-	rml.reserve(1000 * num_rows);
-
-	for (int i = 0; i < num_rows; i++)
-	{
-		int index = rng() % 1000;
-		int route = rng() % 50;
-		int max = (rng() % 40) + 10;
-		int value = rng() % max;
-		Rml::String rml_row = Rml::CreateString(1000, R"(
+static const char* DefaultRow = R"(
 			<div class="row">
 				<div class="col col1"><button class="expand" index="%d">+</button>&nbsp;<a>Route %d</a></div>
 				<div class="col col23"><input type="range" class="assign_range" min="0" max="%d" value="%d"/></div>
@@ -104,11 +91,41 @@ static String GenerateRml(const int num_rows)
 						<input type="submit" class="vehicle_depot_assign_confirm" quantity="0">Confirm</input>
 					</div>
 				</div>
-			</div>)",
-			index, route, max, value);
+			</div>)";
+
+static const char* LongTextRow = R"(
+			<div class="row">
+				<div class="col col1"><button class="expand" index="%d">+</button>&nbsp;<a>Ut pulvinar urna nulla. Donec sed sollicitudin diam. Donec eu mauris massa. Suspendisse facilisis mollis dictum. Curabitur mollis nisi eu est semper, quis ultrices augue facilisis. Quisque venenatis malesuada leo, quis dictum turpis tristique at. Integer ut nunc nec odio imperdiet dignissim. %d</a></div>
+				<div class="col col23"><input type="range" class="assign_range" min="0" max="%d" value="%d"/></div>
+				<div class="col col4">Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam eros neque, blandit aliquam consectetur vitae, ornare ac magna. Nam purus nulla, vestibulum a mi vitae, vestibulum porta dolor. Interdum et malesuada fames ac ante ipsum primis in faucibus. Morbi euismod placerat libero, vel elementum purus blandit a. Aenean sed arcu dictum, pharetra diam tempor, tristique est. Vestibulum sagittis leo nec purus consectetur imperdiet. Aenean dictum, neque vitae consequat egestas, mi nibh rhoncus sapien, eu scelerisque eros arcu non lorem. Suspendisse eu pellentesque velit, non sagittis eros. Maecenas tellus odio, condimentum vitae volutpat at, varius eget leo. Maecenas dignissim sem a ligula fermentum</div>
+				<select>
+					<option>Red</option><option>Blue</option><option selected>Green</option><option style="background-color: yellow;">Yellow</option>
+				</select>
+				<div class="inrow unmark_collapse">
+					<div class="col col123 assign_text">Quisque rhoncus ante arcu, at dapibus nulla mattis et. Fusce ac lacinia urna. Nulla facilisi. Morbi consequat ligula eget urna congue pellentesque. Nullam a risus mattis lectus rutrum rutrum. Etiam pharetra libero vitae nibh lobortis vestibulum. Fusce malesuada ligula sem, vitae bibendum mi sodales ac. Fusce mollis nunc non urna hendrerit viverra. Praesent ornare nunc dictum turpis suscipit, in lacinia risus malesuada. Sed sollicitudin purus eget sapien elementum venenatis.</div>
+					<div class="col col4">
+						<input type="submit" class="vehicle_depot_assign_confirm" quantity="0">Confirm</input>
+					</div>
+				</div>
+			</div>)";
+
+static String GenerateRml(const int num_rows, const char* row)
+{
+	static nanobench::Rng rng;
+	
+	Rml::String rml;
+	rml.reserve(10000 * num_rows);
+	
+	for (int i = 0; i < num_rows; i++)
+	{
+		int index = rng() % 1000;
+		int route = rng() % 50;
+		int max = (rng() % 40) + 10;
+		int value = rng() % max;
+		Rml::String rml_row = Rml::CreateString(10000, row, index, route, max, value);
 		rml += rml_row;
 	}
-
+	
 	return rml;
 }
 
@@ -124,7 +141,7 @@ TEST_CASE("element.creation_and_destruction")
 	Element* el = document->GetElementById("performance");
 	REQUIRE(el);
 	constexpr int num_rows = 50;
-	const String rml = GenerateRml(num_rows);
+	const String rml = GenerateRml(num_rows, DefaultRow);
 
 	el->SetInnerRML(rml);
 	context->Update();
@@ -175,6 +192,69 @@ TEST_CASE("element.creation_and_destruction")
 	document->Close();
 }
 
+TEST_CASE("element.long_texts")
+{
+	Context* context = TestsShell::GetContext();
+	REQUIRE(context);
+	
+	ElementDocument* document = context->LoadDocumentFromMemory(document_rml);
+	REQUIRE(document);
+	document->Show();
+	
+	Element* el = document->GetElementById("performance");
+	REQUIRE(el);
+	constexpr int num_rows = 50;
+	const String rml = GenerateRml(num_rows, LongTextRow);
+	
+	el->SetInnerRML(rml);
+	context->Update();
+	context->Render();
+	TestsShell::RenderLoop();
+	
+	String msg = Rml::CreateString(128, "\nElement construction and destruction of %d total very long elements.\n", GetNumDescendentElements(el));
+	msg += TestsShell::GetRenderStats();
+	MESSAGE(msg);
+	
+	nanobench::Bench bench;
+	bench.title("Element");
+	bench.timeUnit(std::chrono::microseconds(1), "us");
+	bench.relative(true);
+	
+	bench.run("Update (unmodified)", [&] { context->Update(); });
+	
+	bool hover_toggle = true;
+	auto child = el->GetChild(num_rows / 2);
+	
+	bench.run("Update (hover child)", [&] {
+		static nanobench::Rng rng;
+		child->SetPseudoClass(":hover", hover_toggle);
+		hover_toggle = !hover_toggle;
+		context->Update();
+	});
+	bench.run("Update (hover)", [&] {
+		el->SetPseudoClass(":hover", hover_toggle);
+		hover_toggle = !hover_toggle;
+		context->Update();
+	});
+	
+	bench.run("Render", [&] { context->Render(); });
+	
+	bench.run("SetInnerRML", [&] { el->SetInnerRML(rml); });
+	
+	bench.run("SetInnerRML + Update", [&] {
+		el->SetInnerRML(rml);
+		context->Update();
+	});
+	
+	bench.run("SetInnerRML + Update + Render", [&] {
+		el->SetInnerRML(rml);
+		context->Update();
+		context->Render();
+	});
+	
+	document->Close();
+}
+
 TEST_CASE("element.asymptotic_complexity")
 {
 	Context* context = TestsShell::GetContext();
@@ -219,7 +299,7 @@ TEST_CASE("element.asymptotic_complexity")
 		// Running the benchmark multiple times, with different number of rows.
 		for (const int num_rows : {1, 2, 5, 10, 20, 50, 100, 200, 500})
 		{
-			const String rml = GenerateRml(num_rows);
+			const String rml = GenerateRml(num_rows, DefaultRow);
 
 			el->SetInnerRML(rml);
 			context->Update();
@@ -234,4 +314,4 @@ TEST_CASE("element.asymptotic_complexity")
 	}
 
 	document->Close();
-}
+}

Tests/Source/UnitTests/XMLParser.cpp

@@ -31,6 +31,7 @@
 #include <RmlUi/Core/Context.h>
 #include <RmlUi/Core/Element.h>
 #include <RmlUi/Core/ElementDocument.h>
+#include <RmlUi/Core/ElementText.h>
 #include <RmlUi/Core/Factory.h>
 #include <doctest.h>
 
@@ -53,6 +54,34 @@ static const String document_xml_tags_in_css = R"(
 </rml>
 )";
 
+static const String document_escaping = R"(
+<rml>
+    <head>
+	<style>
+	p { 
+		font-family: LatoLatin;
+	}
+	</style>
+    </head>
+    <body>
+	<p id="p">&#x20AC;&#8364;</p>
+    </body>
+</rml>
+)";
+
+static const String document_escaping_tags = R"(
+<rml>
+    <head>
+	<style>
+	* { 
+		font-family: LatoLatin;
+	}
+	</style>
+    </head>
+    <body>&lt;p&gt;&amp;lt;span/&amp;gt;&lt;/p&gt;</body>
+</rml>
+)";
+
 TEST_CASE("XMLParser")
 {
 	Context* context = TestsShell::GetContext();
@@ -75,3 +104,44 @@ TEST_CASE("XMLParser")
 	document->Close();
 	TestsShell::ShutdownShell();
 }
+
+TEST_CASE("XMLParser.escaping")
+{
+	Context* context = TestsShell::GetContext();
+	REQUIRE(context);
+	
+	ElementDocument* document = context->LoadDocumentFromMemory(document_escaping);
+	REQUIRE(document);
+	document->Show();
+	
+	TestsShell::RenderLoop();
+	
+	auto element = document->GetElementById("p");
+	REQUIRE(element);
+	
+	CHECK(element->GetInnerRML() == "\xe2\x82\xac\xe2\x82\xac");
+	
+	document->Close();
+	TestsShell::ShutdownShell();
+}
+
+TEST_CASE("XMLParser.escaping_tags")
+{
+	Context* context = TestsShell::GetContext();
+	REQUIRE(context);
+	
+	ElementDocument* document = context->LoadDocumentFromMemory(document_escaping_tags);
+	REQUIRE(document);
+	document->Show();
+	
+	TestsShell::RenderLoop();
+	
+	CHECK(document->GetNumChildren() == 1); 
+	CHECK(document->GetFirstChild()->GetTagName() == "#text");
+	// Text-access should yield decoded value, while RML-access should yield encoded value
+	CHECK(static_cast<ElementText*>(document->GetFirstChild())->GetText() == "<p>&lt;span/&gt;</p>");
+	CHECK(document->GetInnerRML() == "&lt;p&gt;&amp;lt;span/&amp;gt;&lt;/p&gt;");
+	
+	document->Close();
+	TestsShell::ShutdownShell();
+}