Made a filesystem wrapper.

Source/DFPSR/api/fileAPI.cpp

@@ -25,6 +25,9 @@
 #include <cstdlib>
 #include "fileAPI.h"
 #include "bufferAPI.h"
+#ifdef __cpp_lib_filesystem
+	#include <filesystem>
+#endif
 
 namespace dsr {
 
@@ -35,18 +38,30 @@ namespace dsr {
 #if defined(WIN32) || defined(_WIN32)
 	#include <windows.h>
 	static const char32_t* pathSeparator = U"\\";
-	static FILE* accessFile(const ReadableString &filename, bool write) {
-		Buffer pathBuffer = string_saveToMemory(filename, CharacterEncoding::BOM_UTF16LE, LineEncoding::CrLf, false, true);
-		return _wfopen((const wchar_t*)buffer_dangerous_getUnsafeData(pathBuffer), write ? L"wb" : L"rb");
-	}
+	static const CharacterEncoding nativeEncoding = CharacterEncoding::BOM_UTF16LE;
+	#define FILE_ACCESS_FUNCTION _wfopen
+	#define FILE_ACCESS_SELECTION (write ? L"wb" : L"rb")
 #else
 	static const char32_t* pathSeparator = U"/";
-	static FILE* accessFile(const ReadableString &filename, bool write) {
-		Buffer pathBuffer = string_saveToMemory(filename, CharacterEncoding::BOM_UTF8, LineEncoding::CrLf, false, true);
-		return fopen((const char*)buffer_dangerous_getUnsafeData(pathBuffer), write ? "wb" : "rb");
-	}
+	static const CharacterEncoding nativeEncoding = CharacterEncoding::BOM_UTF8;
+	#define FILE_ACCESS_FUNCTION fopen
+	#define FILE_ACCESS_SELECTION (write ? "wb" : "rb")
 #endif
 
+static const NativeChar* toNativeString(const ReadableString &filename, Buffer &buffer) {
+	buffer = string_saveToMemory(filename, nativeEncoding, LineEncoding::CrLf, false, true);
+	return (const NativeChar*)buffer_dangerous_getUnsafeData(buffer);
+}
+
+static String fromNativeString(NativeChar *text) {
+	return string_dangerous_decodeFromData(text, nativeEncoding);
+}
+
+static FILE* accessFile(const ReadableString &filename, bool write) {
+	Buffer buffer;
+	return FILE_ACCESS_FUNCTION(toNativeString(filename, buffer), FILE_ACCESS_SELECTION);
+}
+
 Buffer file_loadBuffer(const ReadableString& filename, bool mustExist) {
 	FILE *file = accessFile(filename, false);
 	if (file != nullptr) {
@@ -86,4 +101,77 @@ const char32_t* file_separator() {
 	return pathSeparator;
 }
 
+ReadableString file_getPathlessName(const ReadableString &path) {
+	int lastSeparator = -1;
+	for (int64_t i = string_length(path) - 1; i >= 0; i--) {
+		DsrChar c = path[i];
+		if (c == U'\\' || c == U'/') {
+			lastSeparator = i;
+			break;
+		}
+	}
+	return string_after(path, lastSeparator);
+}
+
+List<String> file_convertInputArguments(int argn, NativeChar **argv) {
+	List<String> result;
+	result.reserve(argn);
+	for (int a = 0; a < argn; a++) {
+		result.push(fromNativeString(argv[a]));
+	}
+	return result;
+}
+
+#ifdef __cpp_lib_filesystem
+
+static String fromU32String(std::u32string text) {
+	dsr::String result;
+	string_reserve(result, text.length());
+	int i = 0;
+	while (true) {
+		DsrChar c = text[i];
+		if (c == '\0') {
+			break;
+		} else {
+			string_appendChar(result, c);
+		}
+		i++;
+	}
+	return result;
+}
+
+#define FROM_PATH(CONTENT) fromU32String((CONTENT).generic_u32string())
+
+// Macros for creating wrapper functions
+#define DEF_FUNC_VOID_TO_STRING(FUNC_NAME, CALL_NAME) String FUNC_NAME() { return FROM_PATH(CALL_NAME()); }
+#define DEF_FUNC_STRING_TO_STRING(FUNC_NAME, ARG_NAME, CALL_NAME) String FUNC_NAME(const ReadableString& ARG_NAME) { Buffer buffer; return FROM_PATH(CALL_NAME(toNativeString(ARG_NAME, buffer))); }
+#define DEF_FUNC_STRING_TO_OTHER(FUNC_NAME, RETURN_TYPE, ARG_NAME, CALL_NAME) RETURN_TYPE FUNC_NAME(const ReadableString& ARG_NAME) { Buffer buffer; return CALL_NAME(toNativeString(ARG_NAME, buffer)); }
+
+// Wrapper function implementations
+DEF_FUNC_VOID_TO_STRING(file_getCurrentPath, std::filesystem::current_path)
+DEF_FUNC_STRING_TO_STRING(file_getAbsolutePath, path, std::filesystem::absolute)
+DEF_FUNC_STRING_TO_STRING(file_getCanonicalPath, path, std::filesystem::canonical)
+DEF_FUNC_STRING_TO_OTHER(file_exists, bool, path, std::filesystem::exists)
+DEF_FUNC_STRING_TO_OTHER(file_getSize, int64_t, path, std::filesystem::file_size)
+DEF_FUNC_STRING_TO_OTHER(file_remove, bool, path, std::filesystem::remove)
+DEF_FUNC_STRING_TO_OTHER(file_removeRecursively, bool, path, std::filesystem::remove_all)
+DEF_FUNC_STRING_TO_OTHER(file_createFolder, bool, path, std::filesystem::create_directory)
+
+void file_getFolderContent(const ReadableString& folderPath, std::function<void(ReadableString, EntryType)> action) {
+	Buffer buffer;
+	for (auto const& entry : std::filesystem::directory_iterator{toNativeString(folderPath, buffer)}) {
+		EntryType entryType = EntryType::Unknown;
+		if (entry.is_directory()) {
+			entryType = EntryType::Folder;
+		} else if (entry.is_regular_file()) {
+			entryType = EntryType::File;
+		} else if (entry.is_symlink()) {
+			entryType = EntryType::SymbolicLink;
+		}
+		action(FROM_PATH(entry.path()), entryType);
+	}
+}
+
+#endif
+
 }

Source/DFPSR/api/fileAPI.h

@@ -26,11 +26,19 @@
 
 #include "../api/stringAPI.h"
 #include "bufferAPI.h"
+#include <version>
 
 // A module for file access that exists to prevent cyclic dependencies between strings and buffers.
 //   Buffers need a filename to be saved or loaded while strings use buffers to store their characters.
-
 namespace dsr {
+	// NativeChar is defined as whatever type the native system uses for expressing Unicode
+	// Use for input arguments in the main function to allow using file_convertInputArguments.
+	#if defined(WIN32) || defined(_WIN32)
+		using NativeChar = wchar_t; // UTF-16
+	#else
+		using NativeChar = char; // UTF-8
+	#endif
+
 	// Post-condition:
 	//   Returns the content of the file referred to be filename.
 	//   If mustExist is true, then failure to load will throw an exception.
@@ -44,6 +52,45 @@ namespace dsr {
 	// Get a path separator for the target operating system.
 	//   Can be used to construct a file path that works for both forward and backward slash separators.
 	const char32_t* file_separator();
+
+	// Returns the local name after the last path separator, or the whole path if no separator was found.
+	ReadableString file_getPathlessName(const ReadableString &path);
+
+	// Convert input arguments from main into a list of dsr::String.
+	// argv must be declared as wchar_t** on MS-Windows to allow using Unicode, which can be done by using the dsr::NativeChar alias:
+	//     int main(int argn, NativeChar **argv) {
+	List<String> file_convertInputArguments(int argn, NativeChar **argv);
+
+	// Portable wrapper over the STD filesystem library, so that you get Unicode support with one type of string on all platforms.
+	// To enable this wrapper, just compile everything with C++17 or a newer version that still has the feature.
+	// Using more STD functionality can make it harder to port your project when C++ compilers are no longer maintained,
+	//   but exploring the filesystem is necessary for selecting files to load or save in editors.
+	#ifdef __cpp_lib_filesystem
+		enum class EntryType {
+			Unknown, Folder, File, SymbolicLink
+		};
+		// Get the current path, from where the application was called and relative paths start.
+		String file_getCurrentPath();
+		// Get the path's absolute form to know the exact location.
+		String file_getAbsolutePath(const ReadableString& path);
+		// Get the path's absolute form without redundancy.
+		String file_getCanonicalPath(const ReadableString& path);
+		// Gets a callback for each file and folder directly inside of folderPath.
+		// Calls back with the entry's path and an integer representing the entry type.
+		void file_getFolderContent(const ReadableString& folderPath, std::function<void(ReadableString, EntryType)> action);
+		// Returns true iff path refers to a valid file or folder.
+		bool file_exists(const ReadableString& path);
+		// Returns the file's size.
+		int64_t file_getSize(const ReadableString& path);
+		// Removes the file or empty folder.
+		// Returns true iff path existed.
+		bool file_remove(const ReadableString& path);
+		// Removes the file or folder including any content.
+		// Returns true iff path existed.
+		bool file_removeRecursively(const ReadableString& path);
+		// Returns true on success.
+		bool file_createFolder(const ReadableString& path);
+	#endif
 }
 
 #endif

Source/DFPSR/api/stringAPI.cpp

@@ -416,18 +416,24 @@ static void feedCharacter(const UTF32WriterFunction &reciever, DsrChar character
 }
 
 // Appends the content of buffer as a BOM-free Latin-1 file into target
-static void feedStringFromFileBuffer_Latin1(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength) {
-	for (int64_t i = 0; i < fileLength; i++) {
+// fileLength is ignored when nullTerminated is true
+template <bool nullTerminated>
+static void feedStringFromFileBuffer_Latin1(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength = 0) {
+	for (int64_t i = 0; i < fileLength || nullTerminated; i++) {
 		DsrChar character = (DsrChar)(buffer[i]);
+		if (nullTerminated && character == 0) { return; }
 		feedCharacter(reciever, character);
 	}
 }
 // Appends the content of buffer as a BOM-free UTF-8 file into target
-static void feedStringFromFileBuffer_UTF8(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength) {
-	for (int64_t i = 0; i < fileLength; i++) {
+// fileLength is ignored when nullTerminated is true
+template <bool nullTerminated>
+static void feedStringFromFileBuffer_UTF8(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength = 0) {
+	for (int64_t i = 0; i < fileLength || nullTerminated; i++) {
 		uint8_t byteA = buffer[i];
 		if (byteA < (uint32_t)0b10000000) {
 			// Single byte (1xxxxxxx)
+			if (nullTerminated && byteA == 0) { return; }
 			feedCharacter(reciever, (DsrChar)byteA);
 		} else {
 			uint32_t character = 0;
@@ -455,6 +461,7 @@ static void feedStringFromFileBuffer_UTF8(const UTF32WriterFunction &reciever, c
 				character = (character << 6) | (nextByte & 0b00111111);
 				extraBytes--;
 			}
+			if (nullTerminated && character == 0) { return; }
 			feedCharacter(reciever, (DsrChar)character);
 		}
 	}
@@ -471,10 +478,11 @@ uint16_t read16bits(const uint8_t* buffer, int64_t startOffset) {
 	}
 }
 
-// Appends the content of buffer as a BOM-free UTF-16 file into target
-template <bool LittleEndian>
-static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength) {
-	for (int64_t i = 0; i < fileLength; i += 2) {
+// Appends the content of buffer as a BOM-free UTF-16 file into target as UTF-32
+// fileLength is ignored when nullTerminated is true
+template <bool LittleEndian, bool nullTerminated>
+static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength = 0) {
+	for (int64_t i = 0; i < fileLength || nullTerminated; i += 2) {
 		// Read the first 16-bit word
 		uint16_t wordA = read16bits<LittleEndian>(buffer, i);
 		// Check if another word is needed
@@ -482,6 +490,7 @@ static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever,
 		//   we can just check if it's within the range reserved for 32-bit encoding
 		if (wordA <= 0xD7FF || wordA >= 0xE000) {
 			// Not in the reserved range, just a single 16-bit character
+			if (nullTerminated && wordA == 0) { return; }
 			feedCharacter(reciever, (DsrChar)wordA);
 		} else {
 			// The given range was reserved and therefore using 32 bits
@@ -489,19 +498,22 @@ static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever,
 			uint16_t wordB = read16bits<LittleEndian>(buffer, i);
 			uint32_t higher10Bits = wordA & (uint32_t)0b1111111111;
 			uint32_t lower10Bits  = wordB & (uint32_t)0b1111111111;
-			feedCharacter(reciever, (DsrChar)(((higher10Bits << 10) | lower10Bits) + (uint32_t)0x10000));
+			DsrChar finalChar = (DsrChar)(((higher10Bits << 10) | lower10Bits) + (uint32_t)0x10000);
+			if (nullTerminated && finalChar == 0) { return; }
+			feedCharacter(reciever, finalChar);
 		}
 	}
 }
-// Appends the content of buffer as a text file of unknown format into target
+// Sends the decoded UTF-32 characters from the encoded buffer into target.
+// The text encoding should be specified using a BOM at the start of buffer, otherwise Latin-1 is assumed.
 static void feedStringFromFileBuffer(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength) {
 	// After removing the BOM bytes, the rest can be seen as a BOM-free text file with a known format
 	if (fileLength >= 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) { // UTF-8
-		feedStringFromFileBuffer_UTF8(reciever, buffer + 3, fileLength - 3);
+		feedStringFromFileBuffer_UTF8<false>(reciever, buffer + 3, fileLength - 3);
 	} else if (fileLength >= 2 && buffer[0] == 0xFE && buffer[1] == 0xFF) { // UTF-16 BE
-		feedStringFromFileBuffer_UTF16<false>(reciever, buffer + 2, fileLength - 2);
+		feedStringFromFileBuffer_UTF16<false, false>(reciever, buffer + 2, fileLength - 2);
 	} else if (fileLength >= 2 && buffer[0] == 0xFF && buffer[1] == 0xFE) { // UTF-16 LE
-		feedStringFromFileBuffer_UTF16<true>(reciever, buffer + 2, fileLength - 2);
+		feedStringFromFileBuffer_UTF16<true, false>(reciever, buffer + 2, fileLength - 2);
 	} else if (fileLength >= 4 && buffer[0] == 0x00 && buffer[1] == 0x00 && buffer[2] == 0xFE && buffer[3] == 0xFF) { // UTF-32 BE
 		//feedStringFromFileBuffer_UTF32BE(receiver, buffer + 4, fileLength - 4);
 		throwError(U"UTF-32 BE format is not yet supported!\n");
@@ -522,10 +534,44 @@ static void feedStringFromFileBuffer(const UTF32WriterFunction &reciever, const
 		throwError(U"UTF-7 format is not yet supported!\n");
 	} else {
 		// No BOM detected, assuming Latin-1 (because it directly corresponds to a unicode sub-set)
-		feedStringFromFileBuffer_Latin1(reciever, buffer, fileLength);
+		feedStringFromFileBuffer_Latin1<false>(reciever, buffer, fileLength);
+	}
+}
+
+// Sends the decoded UTF-32 characters from the encoded null terminated buffer into target.
+// buffer may not contain any BOM, and must be null terminated in the specified encoding.
+static void feedStringFromRawData(const UTF32WriterFunction &reciever, const uint8_t* buffer, CharacterEncoding encoding) {
+	if (encoding == CharacterEncoding::Raw_Latin1) {
+		feedStringFromFileBuffer_Latin1<true>(reciever, buffer);
+	} else if (encoding == CharacterEncoding::BOM_UTF8) {
+		feedStringFromFileBuffer_UTF8<true>(reciever, buffer);
+	} else if (encoding == CharacterEncoding::BOM_UTF16BE) {
+		feedStringFromFileBuffer_UTF16<false, true>(reciever, buffer);
+	} else if (encoding == CharacterEncoding::BOM_UTF16LE) {
+		feedStringFromFileBuffer_UTF16<true, true>(reciever, buffer);
+	} else {
+		throwError("Unhandled encoding in feedStringFromRawData!\n");
 	}
 }
 
+String dsr::string_dangerous_decodeFromData(const void* data, CharacterEncoding encoding) {
+	String result;
+	// Measure the size of the result by scanning the content in advance
+	int64_t characterCount = 0;
+	UTF32WriterFunction measurer = [&characterCount](DsrChar character) {
+		characterCount++;
+	};
+	feedStringFromRawData(measurer, (const uint8_t*)data, encoding);
+	// Pre-allocate the correct amount of memory based on the simulation
+	string_reserve(result, characterCount);
+	// Stream output to the result string
+	UTF32WriterFunction reciever = [&result](DsrChar character) {
+		string_appendChar(result, character);
+	};
+	feedStringFromRawData(reciever, (const uint8_t*)data, encoding);
+	return result;
+}
+
 String dsr::string_loadFromMemory(Buffer fileContent) {
 	String result;
 	// Measure the size of the result by scanning the content in advance

Source/DFPSR/api/stringAPI.h

@@ -279,6 +279,13 @@ double string_toDouble(const ReadableString& source);
 String string_load(const ReadableString& filename, bool mustExist = true);
 // Decode a text file from a buffer, which can be loaded using buffer_load.
 String string_loadFromMemory(Buffer fileContent);
+// Decode a null terminated string without BOM, by specifying which format it was encoded in.
+// Pre-conditions:
+//   data does not start with any byte-order-mark (BOM).
+//   data must be null terminated with '\0' in whatever format is being used. Otherwise you may have random crashes
+// Post-condition:
+//   Returns a string decoded from the raw data.
+String string_dangerous_decodeFromData(const void* data, CharacterEncoding encoding);
 
 // Side-effect: Saves content to filename using the selected character and line encodings.
 // Do not add carriage return characters yourself into strings, for these will be added automatically in the CrLf mode.

Source/SDK/fileFinder/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Assuming that you called build.sh from its own folder, you should already be in the project folder.
+PROJECT_FOLDER=.
+# Placing your executable in the project folder allow using the same relative paths in the final release.
+TARGET_FILE=./application
+# The root folder is where DFPSR, SDK and tools are located.
+ROOT_PATH=../..
+# Select where to place temporary files.
+TEMP_DIR=${ROOT_PATH}/../../temporary
+# Select a window manager
+WINDOW_MANAGER=NONE
+# Select safe debug mode or fast release mode
+#MODE=-DDEBUG #Debug mode
+MODE=-DNDEBUG #Release mode
+COMPILER_FLAGS="${MODE} -std=c++17 -O2"
+# Select external libraries
+LINKER_FLAGS=""
+
+# Give execution permission
+chmod +x ${ROOT_PATH}/tools/buildAndRun.sh;
+# Compile everything
+${ROOT_PATH}/tools/build.sh "${PROJECT_FOLDER}" "${TARGET_FILE}" "${ROOT_PATH}" "${TEMP_DIR}" "${WINDOW_MANAGER}" "${COMPILER_FLAGS}" "${LINKER_FLAGS}";
+# Run the application
+${TARGET_FILE} ./পরীক্ষা;

Source/SDK/fileFinder/main.cpp

@@ -0,0 +1,50 @@
+
+// A file finding application showing how to use the filesystem wrapper in the file API using C++ 2017.
+
+/*
+TODO:
+ * Function for getting EntryType from a path.
+ * Implement file_getParent, getting the parent folder or root path, so that it's easy to iterate the other way.
+ * Something for listing root paths, so that systems with more than one system root can have a listbox with drives to select in file explorers.
+ * Wrap file_getPermissions
+ * Wrap file_moveAndRename (over std::filesystem::rename)
+ * Wrap copy (over std::filesystem::copy)
+*/
+
+#include "../../DFPSR/includeFramework.h"
+
+using namespace dsr;
+
+void exploreFolder(const ReadableString& folderPath, const ReadableString& indentation) {
+	file_getFolderContent(folderPath, [indentation](ReadableString entryPath, EntryType entryType) {
+		ReadableString shortName = file_getPathlessName(entryPath);
+		if (entryType == EntryType::Folder) {
+			printText(indentation, " Folder(", shortName, ")\n");
+			exploreFolder(entryPath, indentation + "  ");
+		} else if (entryType == EntryType::File) {
+			printText(indentation, " File(", shortName, ") of ", file_getSize(entryPath), " bytes\n");
+		}
+	});
+}
+
+int main(int argn, NativeChar **argv) {
+	List<String> args = file_convertInputArguments(argn, argv);
+	printText("Input arguments:\n");
+	for (int a = 0; a < args.length(); a++) {
+		printText("  args[", a, "] = ", args[a], "\n");
+	}
+	String absolutePath = file_getCanonicalPath(args[0]);
+	printText("Absolute path = ", absolutePath, "\n");
+	if (args.length() > 1) {
+		// Explore each listed folder from input arguments.
+		for (int a = 1; a < args.length(); a++) {
+			printText("Exploring ", args[a], "\n");
+			exploreFolder(args[a], U"");
+		}
+	} else {
+		// Test program on the current path when nothing was entered.
+		String currentPath = file_getCurrentPath();
+		printText("Exploring ", currentPath, " because no folders were given.\n");
+		exploreFolder(currentPath, U"");
+	}
+}

Source/SDK/fileFinder/পরীক্ষা/Svenska.txt

@@ -0,0 +1,3 @@
+Detta är ett test.
+abcdefghijklmnopqrstuvwxyzåäö
+ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖ

Source/SDK/fileFinder/পরীক্ষা/Թեստ/Թեստ.txt

@@ -0,0 +1,2 @@
+Խելագար շները կիրակի օրը ձու են ուտում
+Ջրի սառեցումը եւ կովերը սովամահ են լինում

Source/SDK/fileFinder/পরীক্ষা/Թեստ/Ֆայլեր/Փաստաթուղթ.txt

@@ -0,0 +1 @@
+Ողջույն

Source/SDK/fileFinder/পরীক্ষা/测试/公文.txt

@@ -0,0 +1 @@
+ţ˙O`Y}˙bv„gSË

Source/templates/basic3D/build.sh

@@ -13,7 +13,7 @@ WINDOW_MANAGER=X11
 # Select safe debug mode or fast release mode
 #MODE=-DDEBUG #Debug mode
 MODE=-DNDEBUG #Release mode
-COMPILER_FLAGS="${MODE} -std=c++14 -O2"
+COMPILER_FLAGS="${MODE} -std=c++17 -O2"
 # Select external libraries
 LINKER_FLAGS=""
 

Source/templates/basicCLI/build.sh

@@ -13,7 +13,7 @@ WINDOW_MANAGER=NONE
 # Select safe debug mode or fast release mode
 #MODE=-DDEBUG #Debug mode
 MODE=-DNDEBUG #Release mode
-COMPILER_FLAGS="${MODE} -std=c++14 -O2"
+COMPILER_FLAGS="${MODE} -std=c++17 -O2"
 # Select external libraries
 LINKER_FLAGS=""
 

Source/templates/basicGUI/build.sh

@@ -13,7 +13,7 @@ WINDOW_MANAGER=X11
 # Select safe debug mode or fast release mode
 #MODE=-DDEBUG #Debug mode
 MODE=-DNDEBUG #Release mode
-COMPILER_FLAGS="${MODE} -std=c++14 -O2"
+COMPILER_FLAGS="${MODE} -std=c++17 -O2"
 # Select external libraries
 LINKER_FLAGS=""