Renamed string_split to string_dangerous_split and string_split_inPlace to string_dangerous_split_inPlace.

Source/DFPSR/api/configAPI.cpp

@@ -26,7 +26,7 @@
 using namespace dsr;
 
 void dsr::config_parse_ini(const ReadableString& content, ConfigIniCallback receiverLambda) {
-	List<ReadableString> lines = string_split(content, U'\n');
+	List<ReadableString> lines = string_dangerous_split(content, U'\n');
 	String block = U"";
 	for (int l = 0; l < lines.length(); l++) {
 		// Get the current line

Source/DFPSR/base/text.cpp

@@ -867,12 +867,17 @@ void dsr::throwErrorMessage(const String& message) {
 	throw std::runtime_error(message.toStdString());
 }
 
-void dsr::string_split_callback(std::function<void(ReadableString)> action, const ReadableString& source, DsrChar separator) {
+void dsr::string_split_callback(std::function<void(ReadableString)> action, const ReadableString& source, DsrChar separator, bool removeWhiteSpace) {
 	int64_t sectionStart = 0;
 	for (int64_t i = 0; i < source.length; i++) {
 		DsrChar c = source[i];
 		if (c == separator) {
-			action(string_exclusiveRange(source, sectionStart, i));
+			ReadableString element = string_exclusiveRange(source, sectionStart, i);
+			if (removeWhiteSpace) {
+				action(string_removeOuterWhiteSpace(element));
+			} else {
+				action(element);
+			}
 			sectionStart = i + 1;
 		}
 	}
@@ -881,7 +886,20 @@ void dsr::string_split_callback(std::function<void(ReadableString)> action, cons
 	}
 }
 
-void dsr::string_split_inPlace(List<ReadableString> &target, const ReadableString& source, DsrChar separator, bool appendResult) {
+// TODO: Try to implement this using reference counting so that it doesn't have to allocate heap memory
+List<String> dsr::string_split_clone(const ReadableString& source, DsrChar separator, bool removeWhiteSpace) {
+	List<String> result;
+	string_split_callback([&result, removeWhiteSpace](ReadableString element) {
+		if (removeWhiteSpace) {
+			result.push(string_removeOuterWhiteSpace(element));
+		} else {
+			result.push(element);
+		}
+	}, source, separator);
+	return result;
+}
+
+void dsr::string_dangerous_split_inPlace(List<ReadableString> &target, const ReadableString& source, DsrChar separator, bool appendResult) {
 	if (!appendResult) {
 		target.clear();
 	}
@@ -890,9 +908,17 @@ void dsr::string_split_inPlace(List<ReadableString> &target, const ReadableStrin
 	}, source, separator);
 }
 
-List<ReadableString> dsr::string_split(const ReadableString& source, DsrChar separator) {
+List<ReadableString> dsr::string_dangerous_split(const ReadableString& source, DsrChar separator) {
 	List<ReadableString> result;
-	string_split_inPlace(result, source, separator);
+	string_dangerous_split_inPlace(result, source, separator);
+	return result;
+}
+
+int64_t dsr::string_splitCount(const ReadableString& source, DsrChar separator) {
+	int64_t result;
+	string_split_callback([&result](ReadableString element) {
+		result++;
+	}, source, separator);
 	return result;
 }
 

Source/DFPSR/base/text.h

@@ -61,7 +61,8 @@ enum class LineEncoding {
 
 // Replacing String with a ReadableString reference for input arguments can make passing of U"" literals faster.
 //   Unlike String, it cannot be constructed from a "" literal, because UTF-32 is used internally.
-//   Trying to assign String onto ReadableString by value will fail because String contains more members
+// Only use by reference for input arguments or to hold temporary results!
+//   A ReadableString created as a sub-string from String will stop working once the parent String is freed.
 class ReadableString {
 IMPL_ACCESS:
 	// A local pointer to the sub-allocation
@@ -114,7 +115,7 @@ public:
 //     No combined characters allowed, use precomposed instead, so that the strings can guarantee a fixed character size
 class String : public ReadableString {
 IMPL_ACCESS:
-	// A reference counted pointer to the buffer, just to keep the allocation
+	// A reference counted pointer to the buffer to allow passing strings around without having to clone the buffer each time
 	Buffer buffer;
 	// Same as readSection, but with write access
 	char32_t* writeSection = nullptr;
@@ -234,10 +235,25 @@ ReadableString string_from(const ReadableString& source, int64_t inclusiveStart)
 //   Example: string_after(U"0123456789", 5) == U"6789"
 ReadableString string_after(const ReadableString& source, int64_t exclusiveStart);
 
-// TODO: Should string_split and string_split_inPlace be removed now that string_split_callback is both safer and faster?
-//       This would remove the dependency on List, in case that one wants a different container.
-
-// Warning!
+// The safest split implementation
+// Post-condition:
+//   Returns a list of strings from source by splitting along separator.
+//   If removeWhiteSpace is true then surrounding white-space will be removed, otherwise white-space is kept.
+// The separating characters are excluded from the resulting strings.
+// The number of strings returned in the list will equal the number of separating characters plus one, so the result may contain empty strings.
+// Each string in the list clones content to its own dynamic buffer. Use string_split_callback if you don't need long term storage.
+List<String> string_split_clone(const ReadableString& source, DsrChar separator, bool removeWhiteSpace = false);
+// The fastest and most powerful split implementation
+// Split a string without needing a list to store the result.
+//   Call it twice using different lambda functions if you want to count the size before allocating a buffer.
+// Side-effects:
+//   Calls action for each sub-string divided by separator in source.
+void string_split_callback(std::function<void(ReadableString)> action, const ReadableString& source, DsrChar separator, bool removeWhiteSpace = false);
+// An alternative overload for having a very long lambda at the end.
+inline void string_split_callback(const ReadableString& source, DsrChar separator, bool removeWhiteSpace, std::function<void(ReadableString)> action) {
+	string_split_callback(action, source, separator, removeWhiteSpace);
+}
+// Warning! May cause a crash.
 //   Do not use a ReadableString generated by splitting a String past the String's lifetime.
 //   ReadableString does not allocate any heap memory but is only a view for data allocated elsewhere.
 //   Use string_split_callback if you want something safer.
@@ -246,8 +262,8 @@ ReadableString string_after(const ReadableString& source, int64_t exclusiveStart
 // The separating characters are excluded from the resulting strings.
 // The number of strings returned in the list will equal the number of separating characters plus one, so the result may contain empty strings.
 // Each string in the list reuses memory from the input string using reference counting, but the list itself will be allocated.
-List<ReadableString> string_split(const ReadableString& source, DsrChar separator);
-// Warning!
+List<ReadableString> string_dangerous_split(const ReadableString& source, DsrChar separator);
+// Warning! May cause a crash.
 //   Do not use a ReadableString generated by splitting a String past the String's lifetime.
 //   ReadableString does not allocate any heap memory but is only a view for data allocated elsewhere.
 //   Use string_split_callback if you want something safer.
@@ -257,13 +273,10 @@ List<ReadableString> string_split(const ReadableString& source, DsrChar separato
 //   Fills the target list with strings from source by splitting along separator.
 //   If appendResult is false (default), any pre-existing elements in the target list will be cleared before writing the result.
 //   If appendResult is true, the result is appended to the existing target list.
-void string_split_inPlace(List<ReadableString> &target, const ReadableString& source, DsrChar separator, bool appendResult = false);
-// Split a string without needing a list to store the result.
-//   This allow filtering and processing before saving the results,
-//   but makes debugging more difficult and prevents pattern detection.
-// Side-effects:
-//   Calls action for each sub-string divided by separator in source.
-void string_split_callback(std::function<void(ReadableString)> action, const ReadableString& source, DsrChar separator);
+void string_dangerous_split_inPlace(List<ReadableString> &target, const ReadableString& source, DsrChar separator, bool appendResult = false);
+// Split source using separator, only to return the number of splits.
+// Useful for pre-allocation.
+int64_t string_splitCount(const ReadableString& source, DsrChar separator);
 
 // Post-condition: Returns true iff c is a digit.
 //   Digit <- '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'

Source/DFPSR/collection/BoundChecks.cpp

@@ -24,16 +24,18 @@
 
 #include "../base/text.h"
 
-using namespace dsr;
+namespace dsr {
 
-void dsr::nonZeroLengthCheck(int64_t length, const char* property) {
+void nonZeroLengthCheck(int64_t length, const char* property) {
 	if (length <= 0) {
 		throwError(property, " may not be zero!\n");
 	}
 }
 
-void dsr::baseZeroBoundCheck(int64_t index, int64_t length, const char* property) {
+void baseZeroBoundCheck(int64_t index, int64_t length, const char* property) {
 	if (index < 0 || index >= length) {
 		throwError(property, " ", index, " is out of bound 0..", (length - 1), "!\n");
 	}
 }
+
+}

Source/DFPSR/font/Font.h

@@ -24,8 +24,8 @@
 #ifndef DFPSR_GUI_FONT
 #define DFPSR_GUI_FONT
 
-//#include "../persistent/includePersistent.h"
 #include "../api/types.h"
+#include "../collection/List.h"
 
 namespace dsr {
 

Source/DFPSR/image/Color.cpp

@@ -42,17 +42,17 @@ ColorRgbI32 ColorRgbI32::mix(const ColorRgbI32& colorA, const ColorRgbI32& color
 	return (colorA * invWeight) + (colorB * weight);
 }
 ColorRgbI32::ColorRgbI32(const ReadableString &content) : red(0), green(0), blue(0) {
-	List<ReadableString> elements = string_split(content, U',');
-	int givenChannels = elements.length();
-	if (givenChannels >= 1) {
-		this-> red = string_toInteger(elements[0]);
-		if (givenChannels >= 2) {
-			this-> green = string_toInteger(elements[1]);
-			if (givenChannels >= 3) {
-				this-> blue = string_toInteger(elements[2]);
-			}
+	int givenChannels = 0;
+	string_split_callback([this, &givenChannels](ReadableString channelValue) {
+		if (givenChannels == 0) {
+			this->red = string_toInteger(channelValue);
+		} else if (givenChannels == 1) {
+			this->green = string_toInteger(channelValue);
+		} else if (givenChannels == 2) {
+			this->blue = string_toInteger(channelValue);
 		}
-	}
+		givenChannels++;
+	}, content, U',');
 }
 ColorRgbaI32 ColorRgbaI32::saturate() const {
 	int32_t red = this->red;
@@ -74,20 +74,19 @@ ColorRgbaI32 ColorRgbaI32::mix(const ColorRgbaI32& colorA, const ColorRgbaI32& c
 	return (colorA * invWeight) + (colorB * weight);
 }
 ColorRgbaI32::ColorRgbaI32(const ReadableString &content) : red(0), green(0), blue(0), alpha(255) {
-	List<ReadableString> elements = string_split(content, U',');
-	int givenChannels = elements.length();
-	if (givenChannels >= 1) {
-		this-> red = string_toInteger(elements[0]);
-		if (givenChannels >= 2) {
-			this-> green = string_toInteger(elements[1]);
-			if (givenChannels >= 3) {
-				this-> blue = string_toInteger(elements[2]);
-				if (givenChannels >= 4) {
-					this-> alpha = string_toInteger(elements[3]);
-				}
-			}
+	int givenChannels = 0;
+	string_split_callback([this, &givenChannels](ReadableString channelValue) {
+		if (givenChannels == 0) {
+			this->red = string_toInteger(channelValue);
+		} else if (givenChannels == 1) {
+			this->green = string_toInteger(channelValue);
+		} else if (givenChannels == 2) {
+			this->blue = string_toInteger(channelValue);
+		} else if (givenChannels == 3) {
+			this->alpha = string_toInteger(channelValue);
 		}
-	}
+		givenChannels++;
+	}, content, U',');
 }
 
 String& dsr::string_toStreamIndented(String& target, const ColorRgbI32& source, const ReadableString& indentation) {

Source/DFPSR/machine/VirtualMachine.cpp

@@ -35,13 +35,10 @@ VirtualMachine::VirtualMachine(const ReadableString& code, const std::shared_ptr
 		printText("Starting media machine.\n");
 	#endif
 	this->methods.pushConstruct(U"<init>", 0, this->machineTypeCount);
-	List<ReadableString> lines = string_split(code, U'\n');
 	#ifdef VIRTUAL_MACHINE_DEBUG_PRINT
 		printText("Reading assembly.\n");
 	#endif
-	List<ReadableString> arguments; // Re-using the buffer for in-place splitting
-	for (int l = 0; l < lines.length(); l++) {
-		ReadableString currentLine = lines[l];
+	string_split_callback([this](ReadableString currentLine) {
 		// If the line has a comment, then skip everything from #
 		int commentIndex = string_findFirst(currentLine, U'#');
 		if (commentIndex > -1) {
@@ -52,12 +49,12 @@ VirtualMachine::VirtualMachine(const ReadableString& code, const std::shared_ptr
 		if (colonIndex > -1) {
 			ReadableString command = string_removeOuterWhiteSpace(string_before(currentLine, colonIndex));
 			ReadableString argumentLine = string_after(currentLine, colonIndex);
-			string_split_inPlace(arguments, argumentLine, U',');
+			List<String> arguments = string_split_clone(argumentLine, U',', true);
 			this->interpretMachineWord(command, arguments);
 		} else if (string_length(currentLine) > 0) {
 			throwError("Unexpected line \"", currentLine, "\".\n");
 		}
-	}
+	}, code, U'\n');
 	// Calling "<init>" to execute global commands
 	#ifdef VIRTUAL_MACHINE_DEBUG_PRINT
 		printText("Initializing global machine state.\n");
@@ -226,7 +223,7 @@ VMA VirtualMachine::VMAfromText(int methodIndex, const ReadableString& content)
 	}
 }
 
-static ReadableString getArg(const List<ReadableString>& arguments, int32_t index) {
+static ReadableString getArg(const List<String>& arguments, int32_t index) {
 	if (index < 0 || index >= arguments.length()) {
 		return U"";
 	} else {
@@ -253,7 +250,7 @@ void VirtualMachine::addReturnInstruction() {
 		}
 	});
 }
-void VirtualMachine::addCallInstructions(const List<ReadableString>& arguments) {
+void VirtualMachine::addCallInstructions(const List<String>& arguments) {
 	if (arguments.length() < 1) {
 		throwError("Cannot make a call without the name of a method!\n");
 	}
@@ -361,7 +358,7 @@ void VirtualMachine::addCallInstructions(const List<ReadableString>& arguments)
 	}, outputArguments);
 }
 
-void VirtualMachine::interpretMachineWord(const ReadableString& command, const List<ReadableString>& arguments) {
+void VirtualMachine::interpretMachineWord(const ReadableString& command, const List<String>& arguments) {
 	#ifdef VIRTUAL_MACHINE_DEBUG_PRINT
 		printText("interpretMachineWord @", this->machineWords.length(), " ", command, "(");
 		for (int a = 0; a < arguments.length(); a++) {

Source/DFPSR/machine/VirtualMachine.h

@@ -328,12 +328,12 @@ struct VirtualMachine {
 	void addMachineWord(MachineOperation operation, const List<VMA>& args);
 	void addMachineWord(MachineOperation operation);
 	void addReturnInstruction();
-	void addCallInstructions(const List<ReadableString>& arguments);
+	void addCallInstructions(const List<String>& arguments);
 	void interpretCommand(const ReadableString& operation, const List<VMA>& resolvedArguments);
 	Variable* declareVariable_aux(const VMTypeDef& typeDef, int methodIndex, AccessType access, const ReadableString& name, bool initialize, const ReadableString& defaultValueText);
 	Variable* declareVariable(int methodIndex, AccessType access, const ReadableString& type, const ReadableString& name, bool initialize, const ReadableString& defaultValueText);
 	VMA VMAfromText(int methodIndex, const ReadableString& content);
-	void interpretMachineWord(const ReadableString& command, const List<ReadableString>& arguments);
+	void interpretMachineWord(const ReadableString& command, const List<String>& arguments);
 
 	// Run-time debug printing
 	#ifdef VIRTUAL_MACHINE_DEBUG_PRINT

Source/DFPSR/persistent/ClassFactory.cpp

@@ -138,9 +138,7 @@ std::shared_ptr<Persistent> dsr::createPersistentClass(const String &type, bool
 std::shared_ptr<Persistent> dsr::createPersistentClassFromText(const ReadableString &text) {
 	std::shared_ptr<Persistent> rootObject, newObject;
 	List<std::shared_ptr<Persistent>> stack;
-	List<ReadableString> lines = string_split(text, U'\n');
-	for (int l = 0; l < lines.length(); l++) {
-		ReadableString line = lines[l];
+	string_split_callback([&rootObject, &newObject, &stack](ReadableString line) {
 		int equalityIndex = string_findFirst(line, '=');
 		if (equalityIndex > -1) {
 			// Assignment
@@ -176,7 +174,7 @@ std::shared_ptr<Persistent> dsr::createPersistentClassFromText(const ReadableStr
 				}
 			}
 		}
-	}
+	}, text, U'\n');
 	// Return the root component which is null on failure
 	return rootObject;
 }

Source/DFPSR/persistent/ClassFactory.h

@@ -25,6 +25,7 @@
 #define DFPSR_PERSISTENT_CLASSFACTORY
 
 #include "../base/text.h"
+#include "../collection/List.h"
 #include <memory>
 
 namespace dsr {

Source/SDK/sandbox/sprite/spriteAPI.cpp

@@ -34,7 +34,7 @@ struct SpriteConfig {
 				} else if (string_caseInsensitiveMatch(key, U"MaxBound")) {
 					this->maxBound = parseFVector3D(value);
 				} else if (string_caseInsensitiveMatch(key, U"Points")) {
-					List<ReadableString> values = string_split(value, U',');
+					List<ReadableString> values = string_dangerous_split(value, U',');
 					if (values.length() % 3 != 0) {
 						throwError("Points contained ", values.length(), " values, which is not evenly divisible by three!");
 					} else {
@@ -45,7 +45,7 @@ struct SpriteConfig {
 						}
 					}
 				} else if (string_caseInsensitiveMatch(key, U"TriangleIndices")) {
-					List<ReadableString> values = string_split(value, U',');
+					List<ReadableString> values = string_dangerous_split(value, U',');
 					if (values.length() % 3 != 0) {
 						throwError("TriangleIndices contained ", values.length(), " values, which is not evenly divisible by three!");
 					} else {

Source/SDK/sandbox/sprite/spriteAPI.h

@@ -12,7 +12,7 @@ namespace dsr {
 
 // TODO: Make into a constructor for each vector type
 inline FVector3D parseFVector3D(const ReadableString& content) {
-	List<ReadableString> args = string_split(content, U',');
+	List<ReadableString> args = string_dangerous_split(content, U',');
 	if (args.length() != 3) {
 		printText("Expected a vector of three decimal values.\n");
 		return FVector3D();

Source/SDK/sandbox/tool.cpp

@@ -387,7 +387,7 @@ static void loadPlyModel(ParserState& state, const ReadableString& content, bool
 	int startPointIndex = model_getNumberOfPoints(targetModel);
 	int targetPart = shadow ? 0 : state.part;
 	// Split lines
-	List<ReadableString> lines = string_split(content, U'\n');
+	List<ReadableString> lines = string_dangerous_split(content, U'\n');
 	List<PlyElement> elements;
 	bool readingContent = false; // True after passing end_header
 	int elementIndex = -1; // current member of elements
@@ -409,7 +409,7 @@ static void loadPlyModel(ParserState& state, const ReadableString& content, bool
 		// Tokenize the current line
 		ReadableString currentLine = string_removeOuterWhiteSpace(lines[l]);
 		List<ReadableString> tokens;
-		string_split_inPlace(tokens, currentLine, U' ');
+		string_dangerous_split_inPlace(tokens, currentLine, U' ');
 		if (tokens.length() > 0 && !string_caseInsensitiveMatch(tokens[0], U"COMMENT")) {
 			if (readingContent) {
 				// Parse geometry
@@ -723,7 +723,7 @@ static void parse_shape(ParserState& state, List<ReadableString>& args, bool sha
 }
 
 static void parse_dsm(ParserState& state, const ReadableString& content) {
-	List<ReadableString> lines = string_split(content, U'\n');
+	List<ReadableString> lines = string_dangerous_split(content, U'\n');
 	List<ReadableString> args; // Reusing the buffer for in-place splitting of arguments on each line
 	for (int l = 0; l < lines.length(); l++) {
 		// Get the current line
@@ -746,7 +746,7 @@ static void parse_dsm(ParserState& state, const ReadableString& content) {
 			} else if (colonIndex > -1) {
 				ReadableString command = string_removeOuterWhiteSpace(string_before(line, colonIndex));
 				ReadableString argContent = string_after(line, colonIndex);
-				string_split_inPlace(args, argContent, U',');
+				string_dangerous_split_inPlace(args, argContent, U',');
 				for (int a = 0; a < args.length(); a++) {
 					args[a] = string_removeOuterWhiteSpace(args[a]);
 				}

Source/test/tests/StringTest.cpp

@@ -247,31 +247,19 @@ START_TEST(String)
 	ASSERT_EQUAL(string_toInteger(U"123"), 123);
 	ASSERT_EQUAL(string_toDouble(U"123"), 123.0);
 	ASSERT_EQUAL(string_toDouble(U"123.456"), 123.456);
-	{ // Splitting
-		List<ReadableString> result;
-		string_split_inPlace(result, U"a.b.c.d", U'.');
+	{ // Clone splitting
+		List<String> result;
+		result = string_split_clone(U"a . b . c . d", U'.', false);
 		ASSERT_EQUAL(result.length(), 4);
+		ASSERT_MATCH(result[0], U"a ");
+		ASSERT_MATCH(result[1], U" b ");
+		ASSERT_MATCH(result[2], U" c ");
+		ASSERT_MATCH(result[3], U" d");
+		result = string_split_clone(U"a . b .\tc", U'.', true);
+		ASSERT_EQUAL(result.length(), 3);
 		ASSERT_MATCH(result[0], U"a");
 		ASSERT_MATCH(result[1], U"b");
 		ASSERT_MATCH(result[2], U"c");
-		ASSERT_MATCH(result[3], U"d");
-		String content = U"One Two Three";
-		result = string_split(content, U' ');
-		ASSERT_EQUAL(result.length(), 3);
-		ASSERT_MATCH(result[0], U"One");
-		ASSERT_MATCH(result[1], U"Two");
-		ASSERT_MATCH(result[2], U"Three");
-		string_split_inPlace(result, U"Four.Five", U'.', true);
-		ASSERT_EQUAL(result.length(), 5);
-		ASSERT_MATCH(result[0], U"One");
-		ASSERT_MATCH(result[1], U"Two");
-		ASSERT_MATCH(result[2], U"Three");
-		ASSERT_MATCH(result[3], U"Four");
-		ASSERT_MATCH(result[4], U"Five");
-		string_split_inPlace(result, U" 1 | 2 ", U'|');
-		ASSERT_EQUAL(result.length(), 2);
-		ASSERT_MATCH(result[0], U" 1 ");
-		ASSERT_MATCH(result[1], U" 2 ");
 	}
 	{ // Callback splitting
 		String numbers = U"1, 3, 5, 7, 9";