Browse Source

Sorted Builder functions into script interpretation, analysis and script generation.

David Piuva 3 years ago
parent
commit
96e49e7ce4

+ 6 - 111
Source/tools/builder/Machine.cpp

@@ -1,6 +1,5 @@
 
 #include "Machine.h"
-#include "generator.h"
 #include "expression.h"
 #include "../../DFPSR/api/fileAPI.h"
 
@@ -10,21 +9,6 @@ using namespace dsr;
 #define INTEGER_EXPR(FIRST_TOKEN, LAST_TOKEN) expression_interpretAsInteger(STRING_EXPR(FIRST_TOKEN, LAST_TOKEN))
 #define PATH_EXPR(FIRST_TOKEN, LAST_TOKEN) file_getTheoreticalAbsolutePath(STRING_EXPR(FIRST_TOKEN, LAST_TOKEN), fromPath)
 
-Extension extensionFromString(const ReadableString& extensionName) {
-	String upperName = string_upperCase(string_removeOuterWhiteSpace(extensionName));
-	Extension result = Extension::Unknown;
-	if (string_match(upperName, U"H")) {
-		result = Extension::H;
-	} else if (string_match(upperName, U"HPP")) {
-		result = Extension::Hpp;
-	} else if (string_match(upperName, U"C")) {
-		result = Extension::C;
-	} else if (string_match(upperName, U"CPP")) {
-		result = Extension::Cpp;
-	}
-	return result;
-}
-
 int64_t findFlag(const Machine &target, const dsr::ReadableString &key) {
 	for (int64_t f = 0; f < target.variables.length(); f++) {
 		if (string_caseInsensitiveMatch(key, target.variables[f].key)) {
@@ -77,19 +61,6 @@ static String evaluateExpression(Machine &target, List<String> &tokens, int64_t
 	});
 }
 
-static void crawlSource(ProjectContext &context, const dsr::ReadableString &absolutePath) {
-	EntryType pathType = file_getEntryType(absolutePath);
-	if (pathType == EntryType::File) {
-		printText(U"Crawling for source from ", absolutePath, U".\n");
-		analyzeFromFile(context, absolutePath);
-	} else if (pathType == EntryType::Folder) {
-		printText(U"Crawling was given the folder ", absolutePath, U" but a source file was expected!\n");
-	} else if (pathType == EntryType::SymbolicLink) {
-		// Symbolic links can point to both files and folder, so we need to follow it and find out what it really is.
-		crawlSource(context, file_followSymbolicLink(absolutePath));
-	}
-}
-
 // Copy inherited variables from parent to child.
 static void inheritMachine(Machine &child, const Machine &parent) {
 	for (int64_t v = 0; v < parent.variables.length(); v++) {
@@ -138,12 +109,14 @@ static void interpretLine(SessionContext &output, Machine &target, List<String>
 			} else if (string_caseInsensitiveMatch(first, U"build")) {
 				// Build one or more other projects from a project file or folder path, as dependencies.
 				//   Having the same external project built twice during the same session is not allowed.
-				Machine childTarget;
-				inheritMachine(childTarget, target);
+				// Evaluate arguments recursively, but let the analyzer do the work.
+				Machine childSettings;
+				inheritMachine(childSettings, target);
 				String projectPath = file_getTheoreticalAbsolutePath(expression_unwrapIfNeeded(second), fromPath); // Use the second token as the folder path.
-				argumentsToSettings(childTarget, tokens, 2); // Send all tokens after the second token as input arguments to buildProjects.
+				argumentsToSettings(childSettings, tokens, 2); // Send all tokens after the second token as input arguments to buildProjects.
 				printText("Building ", second, " from ", fromPath, " which is ", projectPath, "\n");
-				build(output, projectPath, childTarget);
+				target.otherProjectPaths.push(projectPath);
+				target.otherProjectSettings.push(childSettings);
 			} else if (string_caseInsensitiveMatch(first, U"link")) {
 				// Only the path name itself is needed, so any redundant -l prefixes will be stripped away.
 				String libraryName = STRING_EXPR(1, tokens.length() - 1);
@@ -236,84 +209,6 @@ void evaluateScript(SessionContext &output, Machine &target, const ReadableStrin
 	}
 }
 
-static List<String> initializedProjects;
-// Using a project file path and input arguments.
-void buildProject(SessionContext &output, const ReadableString &projectFilePath, Machine settings) {
-	printText("Building project at ", projectFilePath, "\n");
-	// Check if this project has begun building previously during this session.
-	String absolutePath = file_getAbsolutePath(projectFilePath);
-	for (int64_t p = 0; p < initializedProjects.length(); p++) {
-		if (string_caseInsensitiveMatch(absolutePath, initializedProjects[p])) {
-			throwError(U"Found duplicate requests to build from the same initial script ", absolutePath, U" which could cause non-determinism if different arguments are given to each!\n");
-			return;
-		}
-	}
-	// Remember that building of this project has started.
-	initializedProjects.push(absolutePath);
-	// Evaluate compiler settings while searching for source code mentioned in the project and imported headers.
-	printText(U"Executing project file from ", projectFilePath, U".\n");
-	ProjectContext context;
-	evaluateScript(output, settings, projectFilePath);
-	// Find out where things are located.
-	String projectPath = file_getAbsoluteParentFolder(projectFilePath);
-	// Get the project's name.
-	String projectName = file_getPathlessName(file_getExtensionless(projectFilePath));
-	// If no application path is given, the new executable will be named after the project and placed in the same folder.
-	String fullProgramPath = getFlag(settings, U"ProgramPath", projectName);
-	if (string_length(output.executableExtension) > 0) {
-		string_append(fullProgramPath, output.executableExtension);
-	}
-	// Interpret ProgramPath relative to the project path.
-	fullProgramPath = file_getTheoreticalAbsolutePath(fullProgramPath, projectPath);
-	// If the SkipIfBinaryExists flag is given, we will abort as soon as we have handled its external BuildProjects requests and confirmed that the application exists.
-	if (getFlagAsInteger(settings, U"SkipIfBinaryExists") && file_getEntryType(fullProgramPath) == EntryType::File) {
-		// SkipIfBinaryExists was active and the binary exists, so abort here to avoid redundant work.
-		printText(U"Skipping build of ", projectFilePath, U" because the SkipIfBinaryExists flag was given and ", fullProgramPath, U" was found.\n");
-		return;
-	}
-	// Once we know where the binary is and that it should be built, we can start searching for source code.
-	for (int o = 0; o < settings.crawlOrigins.length(); o++) {
-		crawlSource(context, settings.crawlOrigins[o]);
-	}
-	// Once we are done finding all source files, we can resolve the dependencies to create a graph connected by indices.
-	resolveDependencies(context);
-	if (getFlagAsInteger(settings, U"ListDependencies")) {
-		printDependencies(context);
-	}
-	gatherBuildInstructions(output, context, settings, fullProgramPath);
-}
-
-// Using a folder path and input arguments for all projects.
-void buildProjects(SessionContext &output, const ReadableString &projectFolderPath, Machine &settings) {
-	printText("Building all projects in ", projectFolderPath, "\n");
-	file_getFolderContent(projectFolderPath, [&settings, &output](const ReadableString& entryPath, const ReadableString& entryName, EntryType entryType) {
-		if (entryType == EntryType::Folder) {
-			buildProjects(output, entryPath, settings);
-		} else if (entryType == EntryType::File) {
-			ReadableString extension = string_upperCase(file_getExtension(entryName));
-			if (string_match(extension, U"DSRPROJ")) {
-				buildProject(output, entryPath, settings);
-			}
-		}
-	});
-}
-
-void build(SessionContext &output, const ReadableString &projectPath, Machine &settings) {
-	EntryType entryType = file_getEntryType(projectPath);
-	printText("Building anything at ", projectPath, " which is ", entryType, "\n");
-	if (entryType == EntryType::File) {
-		String extension = string_upperCase(file_getExtension(projectPath));
-		if (!string_match(extension, U"DSRPROJ")) {
-			printText(U"Can't use the Build keyword with a file that is not a project!\n");
-		} else {
-			// Build the given project
-			buildProject(output, projectPath, settings);
-		}
-	} else if (entryType == EntryType::Folder) {
-		buildProjects(output, projectPath, settings);
-	}
-}
-
 void argumentsToSettings(Machine &settings, const List<String> &arguments, int64_t firstArgument) {
 	for (int64_t a = firstArgument; a < arguments.length(); a++) {
 		String argument = arguments[a];

+ 1 - 93
Source/tools/builder/Machine.h

@@ -3,93 +3,10 @@
 #define DSR_BUILDER_MACHINE_MODULE
 
 #include "../../DFPSR/api/stringAPI.h"
+#include "builderTypes.h"
 
 using namespace dsr;
 
-struct Flag {
-	// Flags created externally using argumentsToSettings from either the command line or another project, will be marked as inherited and given to the next call.
-	bool inherited;
-	dsr::String key, value;
-	Flag() {}
-	Flag(const dsr::ReadableString &key, const dsr::ReadableString &value, bool inherited)
-	: key(key), value(value), inherited(inherited) {}
-};
-
-struct Machine {
-	List<Flag> variables;
-	List<String> compilerFlags;
-	List<String> linkerFlags;
-	List<String> crawlOrigins;
-	// When activeStackDepth < currentStackDepth, we are skipping false cases.
-	int64_t currentStackDepth = 0; // How many scopes we are inside of, from the root script including all the others.
-	int64_t activeStackDepth = 0;
-};
-
-enum class Extension {
-	Unknown, H, Hpp, C, Cpp
-};
-
-enum class ScriptLanguage {
-	Unknown,
-	Batch,
-	Bash
-};
-
-Extension extensionFromString(const ReadableString& extensionName);
-
-struct Connection {
-	String path;
-	int64_t lineNumber = -1;
-	int64_t dependencyIndex = -1;
-	Connection(const ReadableString& path)
-	: path(path) {}
-	Connection(const ReadableString& path, int64_t lineNumber)
-	: path(path), lineNumber(lineNumber) {}
-};
-
-struct Dependency {
-	String path;
-	Extension extension;
-	uint64_t contentChecksum;
-	bool visited; // Used to avoid infinite loops while traversing dependencies.
-	List<Connection> links; // Depends on having these linked after compiling.
-	List<Connection> includes; // Depends on having these included in pre-processing.
-	Dependency(const ReadableString& path, Extension extension)
-	: path(path), extension(extension) {}
-};
-
-struct ProjectContext {
-	//Machine settings;
-	List<Dependency> dependencies;
-	ProjectContext() {}
-};
-
-struct SourceObject {
-	uint64_t identityChecksum = 0; // Identification number for the object's name.
-	uint64_t combinedChecksum = 0; // Combined content of the source file and all included headers recursively.
-	String sourcePath, objectPath, generatedCompilerFlags, compilerName, compileFrom;
-	SourceObject(uint64_t identityChecksum, uint64_t combinedChecksum, const ReadableString& sourcePath, const ReadableString& objectPath, const ReadableString& generatedCompilerFlags, const ReadableString& compilerName, const ReadableString& compileFrom)
-	: identityChecksum(identityChecksum), combinedChecksum(combinedChecksum), sourcePath(sourcePath), objectPath(objectPath), generatedCompilerFlags(generatedCompilerFlags), compilerName(compilerName), compileFrom(compileFrom) {}
-};
-
-struct LinkingStep {
-	String compilerName, compileFrom, binaryName;
-	List<String> linkerFlags;
-	List<int64_t> sourceObjectIndices;
-	bool executeResult;
-	LinkingStep(const ReadableString &compilerName, const ReadableString &compileFrom, const ReadableString &binaryName, const List<String> &linkerFlags, const List<int64_t> &sourceObjectIndices, bool executeResult)
-	: compilerName(compilerName), compileFrom(compileFrom), binaryName(binaryName), linkerFlags(linkerFlags), sourceObjectIndices(sourceObjectIndices), executeResult(executeResult) {}
-};
-
-struct SessionContext {
-	String tempPath;
-	String executableExtension;
-	List<SourceObject> sourceObjects;
-	List<LinkingStep> linkerSteps;
-	SessionContext(const ReadableString &tempPath, const ReadableString &executableExtension)
-	: tempPath(tempPath), executableExtension(executableExtension) {}
-};
-
 // Returns the first case insensitive match for key in target, or -1 if not found.
 int64_t findFlag(const Machine &target, const dsr::ReadableString &key);
 // Returns the value of key in target, or defaultValue if not found.
@@ -104,15 +21,6 @@ void assignValue(Machine &target, const dsr::ReadableString &key, const dsr::Rea
 // Recursively including other scripts using the script's folder as the origin for relative paths.
 void evaluateScript(SessionContext &output, Machine &target, const ReadableString &scriptPath);
 
-// Build anything in projectPath.
-void build(SessionContext &output, const ReadableString &projectPath, Machine &settings);
-
-// Build the project in projectFilePath.
-// Settings must be taken by value to prevent side-effects from spilling over between different scripts.
-void buildProject(SessionContext &output, const ReadableString &projectFilePath, Machine settings);
-
-// Build all projects in projectFolderPath.
-void buildProjects(SessionContext &output, const ReadableString &projectFolderPath, Machine &settings);
 void argumentsToSettings(Machine &settings, const List<String> &arguments, int64_t firstArgument);
 
 #endif

+ 457 - 0
Source/tools/builder/analyzer.cpp

@@ -0,0 +1,457 @@
+
+#include "generator.h"
+#include "Machine.h"
+
+using namespace dsr;
+
+static Extension extensionFromString(const ReadableString& extensionName) {
+	String upperName = string_upperCase(string_removeOuterWhiteSpace(extensionName));
+	Extension result = Extension::Unknown;
+	if (string_match(upperName, U"H")) {
+		result = Extension::H;
+	} else if (string_match(upperName, U"HPP")) {
+		result = Extension::Hpp;
+	} else if (string_match(upperName, U"C")) {
+		result = Extension::C;
+	} else if (string_match(upperName, U"CPP")) {
+		result = Extension::Cpp;
+	}
+	return result;
+}
+
+static uint64_t checksum(const ReadableString& text) {
+	uint64_t a = 0x8C2A03D4;
+	uint64_t b = 0xF42B1583;
+	uint64_t c = 0xA6815E74;
+	uint64_t d = 0;
+	for (int64_t i = 0; i < string_length(text); i++) {
+		a = (b * c + ((i * 3756 + 2654) & 58043)) & 0xFFFFFFFF;
+		b = (231 + text[i] * (a & 154) + c * 867 + 28294061) & 0xFFFFFFFF;
+		c = (a ^ b ^ (text[i] * 1543217521)) & 0xFFFFFFFF;
+		d = d ^ (a << 32) ^ b ^ (c << 16);
+	}
+	return d;
+}
+
+static uint64_t checksum(const Buffer& buffer) {
+	SafePointer<uint8_t> data = buffer_getSafeData<uint8_t>(buffer, "checksum input buffer");
+	uint64_t a = 0x8C2A03D4;
+	uint64_t b = 0xF42B1583;
+	uint64_t c = 0xA6815E74;
+	uint64_t d = 0;
+	for (int64_t i = 0; i < buffer_getSize(buffer); i++) {
+		a = (b * c + ((i * 3756 + 2654) & 58043)) & 0xFFFFFFFF;
+		b = (231 + data[i] * (a & 154) + c * 867 + 28294061) & 0xFFFFFFFF;
+		c = (a ^ b ^ (data[i] * 1543217521)) & 0xFFFFFFFF;
+		d = d ^ (a << 32) ^ b ^ (c << 16);
+	}
+	return d;
+}
+
+static int64_t findDependency(ProjectContext &context, const ReadableString& findPath) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
+		if (string_match(context.dependencies[d].path, findPath)) {
+			return d;
+		}
+	}
+	return -1;
+}
+
+static void resolveConnection(ProjectContext &context, Connection &connection) {
+	connection.dependencyIndex = findDependency(context, connection.path);
+}
+
+static void resolveDependency(ProjectContext &context, Dependency &dependency) {
+	for (int64_t l = 0; l < dependency.links.length(); l++) {
+		resolveConnection(context, dependency.links[l]);
+	}
+	for (int64_t i = 0; i < dependency.includes.length(); i++) {
+		resolveConnection(context, dependency.includes[i]);
+	}
+}
+
+void resolveDependencies(ProjectContext &context) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
+		resolveDependency(context, context.dependencies[d]);
+	}
+}
+
+static String findSourceFile(const ReadableString& headerPath, bool acceptC, bool acceptCpp) {
+	if (file_hasExtension(headerPath)) {
+		ReadableString extensionlessPath = file_getExtensionless(headerPath);
+		String cPath = extensionlessPath + U".c";
+		String cppPath = extensionlessPath + U".cpp";
+		if (acceptC && file_getEntryType(cPath) == EntryType::File) {
+			return cPath;
+		} else if (acceptCpp && file_getEntryType(cppPath) == EntryType::File) {
+			return cppPath;
+		}
+	}
+	return U"";
+}
+
+static void flushToken(List<String> &target, String &currentToken) {
+	if (string_length(currentToken) > 0) {
+		target.push(currentToken);
+		currentToken = U"";
+	}
+}
+
+static void tokenize(List<String> &target, const ReadableString& line) {
+	String currentToken;
+	for (int64_t i = 0; i < string_length(line); i++) {
+		DsrChar c = line[i];
+		DsrChar nextC = line[i + 1];
+		if (c == U'#' && nextC == U'#') {
+			// Appending tokens using ##
+			i++;
+		} else if (c == U'#' || c == U'(' || c == U')' || c == U'[' || c == U']' || c == U'{' || c == U'}') {
+			// Atomic token of a single character
+			flushToken(target, currentToken);
+			string_appendChar(currentToken, c);
+			flushToken(target, currentToken);
+		} else if (c == U' ' || c == U'\t') {
+			// Whitespace
+			flushToken(target, currentToken);
+		} else {
+			string_appendChar(currentToken, c);
+		}
+	}
+	flushToken(target, currentToken);
+}
+
+// When CACHED_ANALYSIS is enabled, files will only be analyzed once per session, by remembering them from previous projects.
+//   If features that require a different type of analysis per project are implemented, this can easily be turned off.
+#define CACHED_ANALYSIS
+
+#ifdef CACHED_ANALYSIS
+	// Remembering previous results from analyzing the same files.
+	List<Dependency> analysisCache;
+#endif
+
+void analyzeFile(Dependency &result, const ReadableString& absolutePath, Extension extension) {
+	#ifdef CACHED_ANALYSIS
+		// Check if the file has already been analyzed.
+		for (int c = 0; c < analysisCache.length(); c++) {
+			if (string_match(analysisCache[c].path, absolutePath)) {
+				// Clone all the results to keep projects separate in memory for safety.
+				result = analysisCache[c];
+				return;
+			}
+		}
+	#endif
+	// Get the file's binary content.
+	Buffer fileBuffer = file_loadBuffer(absolutePath);
+	// Get the checksum
+	result.contentChecksum = checksum(fileBuffer);
+	if (extension == Extension::H || extension == Extension::Hpp) {
+		// The current file is a header, so look for an implementation with the corresponding name.
+		String sourcePath = findSourceFile(absolutePath, extension == Extension::H, true);
+		// If found:
+		if (string_length(sourcePath) > 0) {
+			// Remember that anything using the header will have to link with the implementation.
+			result.links.pushConstruct(sourcePath);
+		}
+	}
+	// Interpret the file's content.
+	String sourceCode = string_loadFromMemory(fileBuffer);
+	String parentFolder = file_getRelativeParentFolder(absolutePath);
+	List<String> tokens;
+	bool continuingLine = false;
+	int64_t lineNumber = 0;
+	string_split_callback(sourceCode, U'\n', true, [&result, &parentFolder, &tokens, &continuingLine, &lineNumber](ReadableString line) {
+		lineNumber++;
+		if (line[0] == U'#' || continuingLine) {
+			tokenize(tokens, line);
+			// Continuing pre-processing line using \ at the end.
+			continuingLine = line[string_length(line) - 1] == U'\\';
+		} else {
+			continuingLine = false;
+		}
+		if (!continuingLine && tokens.length() > 0) {
+			if (tokens.length() >= 3) {
+				if (string_match(tokens[1], U"include")) {
+					if (tokens[2][0] == U'\"') {
+						String relativePath = string_unmangleQuote(tokens[2]);
+						String absolutePath = file_getTheoreticalAbsolutePath(relativePath, parentFolder, LOCAL_PATH_SYNTAX);
+						result.includes.pushConstruct(absolutePath, lineNumber);
+					}
+				}
+			}
+			tokens.clear();
+		}
+	});
+}
+
+void analyzeFromFile(ProjectContext &context, const ReadableString& absolutePath) {
+	if (findDependency(context, absolutePath) != -1) {
+		// Already analyzed the current entry. Abort to prevent duplicate dependencies.
+		return;
+	}
+	Extension extension = extensionFromString(file_getExtension(absolutePath));
+	if (extension != Extension::Unknown) {
+		// Create a new dependency for the file.
+		int64_t parentIndex = context.dependencies.length();
+		context.dependencies.push(Dependency(absolutePath, extension));
+		// Summarize the file's content.
+		analyzeFile(context.dependencies[parentIndex], absolutePath, extension);
+		// Continue analyzing recursively into the file's dependencies.
+		for (int64_t i = 0; i < context.dependencies[parentIndex].includes.length(); i++) {
+			analyzeFromFile(context, context.dependencies[parentIndex].includes[i].path);
+		}
+		for (int64_t l = 0; l < context.dependencies[parentIndex].links.length(); l++) {
+			analyzeFromFile(context, context.dependencies[parentIndex].links[l].path);
+		}
+	}
+}
+
+static void debugPrintDependencyList(const List<Connection> &connnections, const ReadableString verb) {
+	for (int64_t c = 0; c < connnections.length(); c++) {
+		int64_t lineNumber = connnections[c].lineNumber;
+		if (lineNumber != -1) {
+			printText(U"  @", lineNumber, U"\t");
+		} else {
+			printText(U"    \t");
+		}
+		printText(U" ", verb, U" ", file_getPathlessName(connnections[c].path), U"\n");
+	}
+}
+
+void printDependencies(ProjectContext &context) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
+		printText(U"* ", file_getPathlessName(context.dependencies[d].path), U"\n");
+		debugPrintDependencyList(context.dependencies[d].includes, U"including");
+		debugPrintDependencyList(context.dependencies[d].links, U"linking");
+	}
+}
+
+static void traverserHeaderChecksums(ProjectContext &context, uint64_t &target, int64_t dependencyIndex) {
+	// Use checksums from headers
+	for (int64_t h = 0; h < context.dependencies[dependencyIndex].includes.length(); h++) {
+		int64_t includedIndex = context.dependencies[dependencyIndex].includes[h].dependencyIndex;
+		if (!context.dependencies[includedIndex].visited) {
+			//printText(U"	traverserHeaderChecksums(context, ", includedIndex, U") ", context.dependencies[includedIndex].path, "\n");
+			// Bitwise exclusive or is both order independent and entropy preserving for non-repeated content.
+			target = target ^ context.dependencies[includedIndex].contentChecksum;
+			// Just have to make sure that the same checksum is not used twice.
+			context.dependencies[includedIndex].visited = true;
+			// Use checksums from headers recursively
+			traverserHeaderChecksums(context, target, includedIndex);
+		}
+	}
+}
+
+static uint64_t getCombinedChecksum(ProjectContext &context, int64_t dependencyIndex) {
+	//printText(U"getCombinedChecksum(context, ", dependencyIndex, U") ", context.dependencies[dependencyIndex].path, "\n");
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
+		context.dependencies[d].visited = false;
+	}
+	context.dependencies[dependencyIndex].visited = true;
+	uint64_t result = context.dependencies[dependencyIndex].contentChecksum;
+	traverserHeaderChecksums(context, result, dependencyIndex);
+	return result;
+}
+
+static int64_t findObject(SessionContext &source, uint64_t identityChecksum) {
+	for (int64_t o = 0; o < source.sourceObjects.length(); o++) {
+		if (source.sourceObjects[o].identityChecksum == identityChecksum) {
+			return o;
+		}
+	}
+	return -1;
+}
+
+void gatherBuildInstructions(SessionContext &output, ProjectContext &context, Machine &settings, ReadableString programPath) {
+	// The compiler is often a global alias, so the user must supply either an alias or an absolute path.
+	ReadableString compilerName = getFlag(settings, U"Compiler", U"g++"); // Assume g++ as the compiler if not specified.
+	ReadableString compileFrom = getFlag(settings, U"CompileFrom", U"");
+	// Check if the build system was asked to run the compiler from a specific folder.
+	bool changePath = (string_length(compileFrom) > 0);
+	if (changePath) {
+		printText(U"Using ", compilerName, " as the compiler executed from ", compileFrom, ".\n");
+	} else {
+		printText(U"Using ", compilerName, " as the compiler from the current directory.\n");
+	}
+	// TODO: Warn if -DNDEBUG, -DDEBUG, or optimization levels are given directly.
+	//       Using the variables instead is both more flexible by accepting input arguments
+	//       and keeping the same format to better reuse compiled objects.
+	if (getFlagAsInteger(settings, U"Debug")) {
+		printText(U"Building with debug mode.\n");
+		settings.compilerFlags.push(U"-DDEBUG");
+	} else {
+		printText(U"Building with release mode.\n");
+		settings.compilerFlags.push(U"-DNDEBUG");
+	}
+	if (getFlagAsInteger(settings, U"StaticRuntime")) {
+		if (getFlagAsInteger(settings, U"Windows")) {
+			printText(U"Building with static runtime. Your application's binary will be bigger but can run without needing any installer.\n");
+			settings.compilerFlags.push(U"-static");
+			settings.compilerFlags.push(U"-static-libgcc");
+			settings.compilerFlags.push(U"-static-libstdc++");
+			settings.linkerFlags.push(U"-static");
+			settings.linkerFlags.push(U"-static-libgcc");
+			settings.linkerFlags.push(U"-static-libstdc++");
+		} else {
+			printText(U"The target platform does not support static linking of runtime. But don't worry about bundling any runtimes, because it comes with most of the Posix compliant operating systems.\n");
+		}
+	} else {
+		printText(U"Building with dynamic runtime. Don't forget to bundle the C and C++ runtimes for systems that don't have it pre-installed.\n");
+	}
+	ReadableString optimizationLevel = getFlag(settings, U"Optimization", U"2");
+		printText(U"Building with optimization level ", optimizationLevel, U".\n");
+	settings.compilerFlags.push(string_combine(U"-O", optimizationLevel));
+
+	// Convert lists of linker and compiler flags into strings.
+	// TODO: Give a warning if two contradictory flags are used, such as optimization levels and language versions.
+	// TODO: Make sure that no spaces are inside of the flags, because that can mess up detection of pre-existing and contradictory arguments.
+	// TODO: Use groups of compiler flags, so that they can be generated in the last step.
+	//       This would allow calling the compiler directly when given a folder path for temporary files instead of a script path.
+	String generatedCompilerFlags;
+	for (int64_t i = 0; i < settings.compilerFlags.length(); i++) {
+		string_append(generatedCompilerFlags, " ", settings.compilerFlags[i]);
+	}
+	String linkerFlags;
+	for (int64_t i = 0; i < settings.linkerFlags.length(); i++) {
+		string_append(linkerFlags, " -l", settings.linkerFlags[i]);
+	}
+	printText(U"Generating build instructions for ", programPath, U" using settings:\n");
+	printText(U"  Compiler flags:", generatedCompilerFlags, U"\n");
+	printText(U"  Linker flags:", linkerFlags, U"\n");
+	for (int64_t v = 0; v < settings.variables.length(); v++) {
+		printText(U"  * ", settings.variables[v].key, U" = ", settings.variables[v].value);
+		if (settings.variables[v].inherited) {
+			printText(U" (inherited input)");
+		}
+		printText(U"\n");
+	}
+	printText(U"Listing source files to compile in the current session.\n");
+	// The current project's global indices to objects shared between all projects being built during the session.
+	List<int64_t> sourceObjectIndices;
+	bool hasSourceCode = false;
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
+		Extension extension = context.dependencies[d].extension;
+		if (extension == Extension::C || extension == Extension::Cpp) {
+			// Dependency paths are already absolute from the recursive search.
+			String sourcePath = context.dependencies[d].path;
+			String identity = string_combine(sourcePath, generatedCompilerFlags);
+			uint64_t identityChecksum = checksum(identity);
+			int64_t previousIndex = findObject(output, identityChecksum);
+			if (previousIndex == -1) {
+				// Content checksums were created while scanning for source code, so now we just combine each source file's content checksum with all its headers to get the combined checksum.
+				// The combined checksum represents the state after all headers are included recursively and given as input for compilation unit generating an object.
+				uint64_t combinedChecksum = getCombinedChecksum(context, d);
+				String objectPath = file_combinePaths(output.tempPath, string_combine(U"dfpsr_", identityChecksum, U"_", combinedChecksum, U".o"));
+				sourceObjectIndices.push(output.sourceObjects.length());
+				output.sourceObjects.pushConstruct(identityChecksum, combinedChecksum, sourcePath, objectPath, generatedCompilerFlags, compilerName, compileFrom);
+			} else {
+				// Link to this pre-existing source file.
+				sourceObjectIndices.push(previousIndex);
+			}
+			hasSourceCode = true;
+		}
+	}
+	if (hasSourceCode) {
+		printText(U"Listing target executable ", programPath, " in the current session.\n");
+		bool executeResult = getFlagAsInteger(settings, U"Supressed") == 0;
+		output.linkerSteps.pushConstruct(compilerName, compileFrom, programPath, settings.linkerFlags, sourceObjectIndices, executeResult);
+	} else {
+		printText(U"Filed to find any source code to compile when building ", programPath, U".\n");
+	}
+}
+
+static void crawlSource(ProjectContext &context, const dsr::ReadableString &absolutePath) {
+	EntryType pathType = file_getEntryType(absolutePath);
+	if (pathType == EntryType::File) {
+		printText(U"Crawling for source from ", absolutePath, U".\n");
+		analyzeFromFile(context, absolutePath);
+	} else if (pathType == EntryType::Folder) {
+		printText(U"Crawling was given the folder ", absolutePath, U" but a source file was expected!\n");
+	} else if (pathType == EntryType::SymbolicLink) {
+		// Symbolic links can point to both files and folder, so we need to follow it and find out what it really is.
+		crawlSource(context, file_followSymbolicLink(absolutePath));
+	}
+}
+
+void build(SessionContext &output, const ReadableString &projectPath, Machine &settings);
+
+static List<String> initializedProjects;
+// Using a project file path and input arguments.
+void buildProject(SessionContext &output, const ReadableString &projectFilePath, Machine settings) {
+	printText("Building project at ", projectFilePath, "\n");
+	// Check if this project has begun building previously during this session.
+	String absolutePath = file_getAbsolutePath(projectFilePath);
+	for (int64_t p = 0; p < initializedProjects.length(); p++) {
+		if (string_caseInsensitiveMatch(absolutePath, initializedProjects[p])) {
+			throwError(U"Found duplicate requests to build from the same initial script ", absolutePath, U" which could cause non-determinism if different arguments are given to each!\n");
+			return;
+		}
+	}
+	// Remember that building of this project has started.
+	initializedProjects.push(absolutePath);
+	// Evaluate compiler settings while searching for source code mentioned in the project and imported headers.
+	printText(U"Executing project file from ", projectFilePath, U".\n");
+	ProjectContext context;
+	evaluateScript(output, settings, projectFilePath);
+	// Find out where things are located.
+	String projectPath = file_getAbsoluteParentFolder(projectFilePath);
+	// Get the project's name.
+	String projectName = file_getPathlessName(file_getExtensionless(projectFilePath));
+	// If no application path is given, the new executable will be named after the project and placed in the same folder.
+	String fullProgramPath = getFlag(settings, U"ProgramPath", projectName);
+	if (string_length(output.executableExtension) > 0) {
+		string_append(fullProgramPath, output.executableExtension);
+	}
+	// Interpret ProgramPath relative to the project path.
+	fullProgramPath = file_getTheoreticalAbsolutePath(fullProgramPath, projectPath);
+	// Build other projects.
+	for (int64_t b = 0; b < settings.otherProjectPaths.length(); b++) {
+		build(output, settings.otherProjectPaths[b], settings.otherProjectSettings[b]);
+	}
+	// If the SkipIfBinaryExists flag is given, we will abort as soon as we have handled its external BuildProjects requests and confirmed that the application exists.
+	if (getFlagAsInteger(settings, U"SkipIfBinaryExists") && file_getEntryType(fullProgramPath) == EntryType::File) {
+		// SkipIfBinaryExists was active and the binary exists, so abort here to avoid redundant work.
+		printText(U"Skipping build of ", projectFilePath, U" because the SkipIfBinaryExists flag was given and ", fullProgramPath, U" was found.\n");
+		return;
+	}
+	// Once we know where the binary is and that it should be built, we can start searching for source code.
+	for (int64_t o = 0; o < settings.crawlOrigins.length(); o++) {
+		crawlSource(context, settings.crawlOrigins[o]);
+	}
+	// Once we are done finding all source files, we can resolve the dependencies to create a graph connected by indices.
+	resolveDependencies(context);
+	if (getFlagAsInteger(settings, U"ListDependencies")) {
+		printDependencies(context);
+	}
+	gatherBuildInstructions(output, context, settings, fullProgramPath);
+}
+
+// Using a folder path and input arguments for all projects.
+void buildProjects(SessionContext &output, const ReadableString &projectFolderPath, Machine &settings) {
+	printText("Building all projects in ", projectFolderPath, "\n");
+	file_getFolderContent(projectFolderPath, [&settings, &output](const ReadableString& entryPath, const ReadableString& entryName, EntryType entryType) {
+		if (entryType == EntryType::Folder) {
+			buildProjects(output, entryPath, settings);
+		} else if (entryType == EntryType::File) {
+			ReadableString extension = string_upperCase(file_getExtension(entryName));
+			if (string_match(extension, U"DSRPROJ")) {
+				buildProject(output, entryPath, settings);
+			}
+		}
+	});
+}
+
+void build(SessionContext &output, const ReadableString &projectPath, Machine &settings) {
+	EntryType entryType = file_getEntryType(projectPath);
+	printText("Building anything at ", projectPath, " which is ", entryType, "\n");
+	if (entryType == EntryType::File) {
+		String extension = string_upperCase(file_getExtension(projectPath));
+		if (!string_match(extension, U"DSRPROJ")) {
+			printText(U"Can't use the Build keyword with a file that is not a project!\n");
+		} else {
+			// Build the given project
+			buildProject(output, projectPath, settings);
+		}
+	} else if (entryType == EntryType::Folder) {
+		buildProjects(output, projectPath, settings);
+	}
+}

+ 30 - 0
Source/tools/builder/analyzer.h

@@ -0,0 +1,30 @@
+
+#ifndef DSR_BUILDER_ANALYZER_MODULE
+#define DSR_BUILDER_ANALYZER_MODULE
+
+#include "../../DFPSR/api/fileAPI.h"
+#include "builderTypes.h"
+
+using namespace dsr;
+
+// Analyze using calls from the machine
+void analyzeFromFile(ProjectContext &context, const ReadableString& entryPath);
+// Call from main when done analyzing source files
+void resolveDependencies(ProjectContext &context);
+
+// Visualize
+void printDependencies(ProjectContext &context);
+
+// Build anything in projectPath.
+void build(SessionContext &output, const ReadableString &projectPath, Machine &settings);
+
+// Build the project in projectFilePath.
+// Settings must be taken by value to prevent side-effects from spilling over between different scripts.
+void buildProject(SessionContext &output, const ReadableString &projectFilePath, Machine settings);
+
+// Build all projects in projectFolderPath.
+void buildProjects(SessionContext &output, const ReadableString &projectFolderPath, Machine &settings);
+
+void gatherBuildInstructions(SessionContext &output, ProjectContext &context, Machine &settings, ReadableString programPath);
+
+#endif

+ 1 - 1
Source/tools/builder/buildProject.bat

@@ -14,7 +14,7 @@ set BUILDER_EXECUTABLE=%BUILDER_FOLDER%builder.exe
 echo BUILDER_EXECUTABLE = %BUILDER_EXECUTABLE%
 set DFPSR_LIBRARY=%BUILDER_FOLDER%..\..\DFPSR
 echo DFPSR_LIBRARY = %DFPSR_LIBRARY%
-set BUILDER_SOURCE=%BUILDER_FOLDER%\main.cpp %BUILDER_FOLDER%\Machine.cpp %BUILDER_FOLDER%\generator.cpp %BUILDER_FOLDER%\expression.cpp %DFPSR_LIBRARY%\collection\collections.cpp %DFPSR_LIBRARY%\api\fileAPI.cpp %DFPSR_LIBRARY%\api\bufferAPI.cpp %DFPSR_LIBRARY%\api\stringAPI.cpp %DFPSR_LIBRARY%\base\SafePointer.cpp
+set BUILDER_SOURCE=%BUILDER_FOLDER%\main.cpp %BUILDER_FOLDER%\Machine.cpp %BUILDER_FOLDER%\generator.cpp %BUILDER_FOLDER%\analyzer.cpp %BUILDER_FOLDER%\expression.cpp %DFPSR_LIBRARY%\collection\collections.cpp %DFPSR_LIBRARY%\api\fileAPI.cpp %DFPSR_LIBRARY%\api\bufferAPI.cpp %DFPSR_LIBRARY%\api\stringAPI.cpp %DFPSR_LIBRARY%\base\SafePointer.cpp
 echo BUILDER_SOURCE = %BUILDER_SOURCE%
 
 set CPP_COMPILER_FOLDER=C:\Program\CodeBlocks\MinGW\bin

+ 1 - 1
Source/tools/builder/buildProject.sh

@@ -20,7 +20,7 @@ if [ -e "${BUILDER_EXECUTABLE}" ]; then
 else
 	echo "Building the Builder build system for first time use."
 	LIBRARY_PATH="$(realpath ${BUILDER_FOLDER}/../../DFPSR)"
-	SOURCE_CODE="${BUILDER_FOLDER}/main.cpp ${BUILDER_FOLDER}/Machine.cpp ${BUILDER_FOLDER}/generator.cpp ${BUILDER_FOLDER}/expression.cpp ${LIBRARY_PATH}/collection/collections.cpp ${LIBRARY_PATH}/api/fileAPI.cpp ${LIBRARY_PATH}/api/bufferAPI.cpp ${LIBRARY_PATH}/api/stringAPI.cpp ${LIBRARY_PATH}/base/SafePointer.cpp"
+	SOURCE_CODE="${BUILDER_FOLDER}/main.cpp ${BUILDER_FOLDER}/Machine.cpp ${BUILDER_FOLDER}/generator.cpp ${BUILDER_FOLDER}/analyzer.cpp ${BUILDER_FOLDER}/expression.cpp ${LIBRARY_PATH}/collection/collections.cpp ${LIBRARY_PATH}/api/fileAPI.cpp ${LIBRARY_PATH}/api/bufferAPI.cpp ${LIBRARY_PATH}/api/stringAPI.cpp ${LIBRARY_PATH}/base/SafePointer.cpp"
 	"${CPP_COMPILER_PATH}" -o "${BUILDER_EXECUTABLE}" ${SOURCE_CODE} -std=c++14
 	if [ $? -eq 0 ]; then
 		echo "Completed building the Builder build system."

+ 93 - 0
Source/tools/builder/builderTypes.h

@@ -0,0 +1,93 @@
+
+#ifndef DSR_BUILDER_TYPES
+#define DSR_BUILDER_TYPES
+
+#include "../../DFPSR/api/stringAPI.h"
+
+using namespace dsr;
+
+struct Flag {
+	// Flags created externally using argumentsToSettings from either the command line or another project, will be marked as inherited and given to the next call.
+	bool inherited;
+	String key, value;
+	Flag() {}
+	Flag(const ReadableString &key, const ReadableString &value, bool inherited)
+	: key(key), value(value), inherited(inherited) {}
+};
+
+struct Machine {
+	List<Flag> variables;
+	List<String> compilerFlags;
+	List<String> linkerFlags;
+	List<String> crawlOrigins;
+	List<String> otherProjectPaths; // Corresponding to otherProjectSettings.
+	List<Machine> otherProjectSettings; // Corresponding to otherProjectPaths.
+	// When activeStackDepth < currentStackDepth, we are skipping false cases.
+	int64_t currentStackDepth = 0; // How many scopes we are inside of, from the root script including all the others.
+	int64_t activeStackDepth = 0;
+};
+
+enum class Extension {
+	Unknown, H, Hpp, C, Cpp
+};
+
+enum class ScriptLanguage {
+	Unknown,
+	Batch,
+	Bash
+};
+
+struct Connection {
+	String path;
+	int64_t lineNumber = -1;
+	int64_t dependencyIndex = -1;
+	Connection(const ReadableString& path)
+	: path(path) {}
+	Connection(const ReadableString& path, int64_t lineNumber)
+	: path(path), lineNumber(lineNumber) {}
+};
+
+struct Dependency {
+	String path;
+	Extension extension;
+	uint64_t contentChecksum;
+	bool visited; // Used to avoid infinite loops while traversing dependencies.
+	List<Connection> links; // Depends on having these linked after compiling.
+	List<Connection> includes; // Depends on having these included in pre-processing.
+	Dependency(const ReadableString& path, Extension extension)
+	: path(path), extension(extension) {}
+};
+
+struct ProjectContext {
+	List<Dependency> dependencies;
+	ProjectContext() {}
+};
+
+struct SourceObject {
+	uint64_t identityChecksum = 0; // Identification number for the object's name.
+	uint64_t combinedChecksum = 0; // Combined content of the source file and all included headers recursively.
+	String sourcePath, objectPath, generatedCompilerFlags, compilerName, compileFrom;
+	SourceObject(uint64_t identityChecksum, uint64_t combinedChecksum, const ReadableString& sourcePath, const ReadableString& objectPath, const ReadableString& generatedCompilerFlags, const ReadableString& compilerName, const ReadableString& compileFrom)
+	: identityChecksum(identityChecksum), combinedChecksum(combinedChecksum), sourcePath(sourcePath), objectPath(objectPath), generatedCompilerFlags(generatedCompilerFlags), compilerName(compilerName), compileFrom(compileFrom) {}
+};
+
+struct LinkingStep {
+	String compilerName, compileFrom, binaryName;
+	List<String> linkerFlags;
+	List<int64_t> sourceObjectIndices;
+	bool executeResult;
+	LinkingStep(const ReadableString &compilerName, const ReadableString &compileFrom, const ReadableString &binaryName, const List<String> &linkerFlags, const List<int64_t> &sourceObjectIndices, bool executeResult)
+	: compilerName(compilerName), compileFrom(compileFrom), binaryName(binaryName), linkerFlags(linkerFlags), sourceObjectIndices(sourceObjectIndices), executeResult(executeResult) {}
+};
+
+struct SessionContext {
+	String tempPath;
+	String executableExtension;
+	List<SourceObject> sourceObjects;
+	List<LinkingStep> linkerSteps;
+	SessionContext(const ReadableString &tempPath, const ReadableString &executableExtension)
+	: tempPath(tempPath), executableExtension(executableExtension) {}
+};
+
+
+#endif

+ 9 - 349
Source/tools/builder/generator.cpp

@@ -3,354 +3,6 @@
 
 using namespace dsr;
 
-static uint64_t checksum(const ReadableString& text) {
-	uint64_t a = 0x8C2A03D4;
-	uint64_t b = 0xF42B1583;
-	uint64_t c = 0xA6815E74;
-	uint64_t d = 0;
-	for (int64_t i = 0; i < string_length(text); i++) {
-		a = (b * c + ((i * 3756 + 2654) & 58043)) & 0xFFFFFFFF;
-		b = (231 + text[i] * (a & 154) + c * 867 + 28294061) & 0xFFFFFFFF;
-		c = (a ^ b ^ (text[i] * 1543217521)) & 0xFFFFFFFF;
-		d = d ^ (a << 32) ^ b ^ (c << 16);
-	}
-	return d;
-}
-
-static uint64_t checksum(const Buffer& buffer) {
-	SafePointer<uint8_t> data = buffer_getSafeData<uint8_t>(buffer, "checksum input buffer");
-	uint64_t a = 0x8C2A03D4;
-	uint64_t b = 0xF42B1583;
-	uint64_t c = 0xA6815E74;
-	uint64_t d = 0;
-	for (int64_t i = 0; i < buffer_getSize(buffer); i++) {
-		a = (b * c + ((i * 3756 + 2654) & 58043)) & 0xFFFFFFFF;
-		b = (231 + data[i] * (a & 154) + c * 867 + 28294061) & 0xFFFFFFFF;
-		c = (a ^ b ^ (data[i] * 1543217521)) & 0xFFFFFFFF;
-		d = d ^ (a << 32) ^ b ^ (c << 16);
-	}
-	return d;
-}
-
-static int64_t findDependency(ProjectContext &context, const ReadableString& findPath) {
-	for (int64_t d = 0; d < context.dependencies.length(); d++) {
-		if (string_match(context.dependencies[d].path, findPath)) {
-			return d;
-		}
-	}
-	return -1;
-}
-
-static void resolveConnection(ProjectContext &context, Connection &connection) {
-	connection.dependencyIndex = findDependency(context, connection.path);
-}
-
-static void resolveDependency(ProjectContext &context, Dependency &dependency) {
-	for (int64_t l = 0; l < dependency.links.length(); l++) {
-		resolveConnection(context, dependency.links[l]);
-	}
-	for (int64_t i = 0; i < dependency.includes.length(); i++) {
-		resolveConnection(context, dependency.includes[i]);
-	}
-}
-
-void resolveDependencies(ProjectContext &context) {
-	for (int64_t d = 0; d < context.dependencies.length(); d++) {
-		resolveDependency(context, context.dependencies[d]);
-	}
-}
-
-static String findSourceFile(const ReadableString& headerPath, bool acceptC, bool acceptCpp) {
-	if (file_hasExtension(headerPath)) {
-		ReadableString extensionlessPath = file_getExtensionless(headerPath);
-		String cPath = extensionlessPath + U".c";
-		String cppPath = extensionlessPath + U".cpp";
-		if (acceptC && file_getEntryType(cPath) == EntryType::File) {
-			return cPath;
-		} else if (acceptCpp && file_getEntryType(cppPath) == EntryType::File) {
-			return cppPath;
-		}
-	}
-	return U"";
-}
-
-static void flushToken(List<String> &target, String &currentToken) {
-	if (string_length(currentToken) > 0) {
-		target.push(currentToken);
-		currentToken = U"";
-	}
-}
-
-static void tokenize(List<String> &target, const ReadableString& line) {
-	String currentToken;
-	for (int64_t i = 0; i < string_length(line); i++) {
-		DsrChar c = line[i];
-		DsrChar nextC = line[i + 1];
-		if (c == U'#' && nextC == U'#') {
-			// Appending tokens using ##
-			i++;
-		} else if (c == U'#' || c == U'(' || c == U')' || c == U'[' || c == U']' || c == U'{' || c == U'}') {
-			// Atomic token of a single character
-			flushToken(target, currentToken);
-			string_appendChar(currentToken, c);
-			flushToken(target, currentToken);
-		} else if (c == U' ' || c == U'\t') {
-			// Whitespace
-			flushToken(target, currentToken);
-		} else {
-			string_appendChar(currentToken, c);
-		}
-	}
-	flushToken(target, currentToken);
-}
-
-// When CACHED_ANALYSIS is enabled, files will only be analyzed once per session, by remembering them from previous projects.
-//   If features that require a different type of analysis per project are implemented, this can easily be turned off.
-#define CACHED_ANALYSIS
-
-#ifdef CACHED_ANALYSIS
-	// Remembering previous results from analyzing the same files.
-	List<Dependency> analysisCache;
-#endif
-
-void analyzeFile(Dependency &result, const ReadableString& absolutePath, Extension extension) {
-	#ifdef CACHED_ANALYSIS
-		// Check if the file has already been analyzed.
-		for (int c = 0; c < analysisCache.length(); c++) {
-			if (string_match(analysisCache[c].path, absolutePath)) {
-				// Clone all the results to keep projects separate in memory for safety.
-				result = analysisCache[c];
-				return;
-			}
-		}
-	#endif
-	// Get the file's binary content.
-	Buffer fileBuffer = file_loadBuffer(absolutePath);
-	// Get the checksum
-	result.contentChecksum = checksum(fileBuffer);
-	if (extension == Extension::H || extension == Extension::Hpp) {
-		// The current file is a header, so look for an implementation with the corresponding name.
-		String sourcePath = findSourceFile(absolutePath, extension == Extension::H, true);
-		// If found:
-		if (string_length(sourcePath) > 0) {
-			// Remember that anything using the header will have to link with the implementation.
-			result.links.pushConstruct(sourcePath);
-		}
-	}
-	// Interpret the file's content.
-	String sourceCode = string_loadFromMemory(fileBuffer);
-	String parentFolder = file_getRelativeParentFolder(absolutePath);
-	List<String> tokens;
-	bool continuingLine = false;
-	int64_t lineNumber = 0;
-	string_split_callback(sourceCode, U'\n', true, [&result, &parentFolder, &tokens, &continuingLine, &lineNumber](ReadableString line) {
-		lineNumber++;
-		if (line[0] == U'#' || continuingLine) {
-			tokenize(tokens, line);
-			// Continuing pre-processing line using \ at the end.
-			continuingLine = line[string_length(line) - 1] == U'\\';
-		} else {
-			continuingLine = false;
-		}
-		if (!continuingLine && tokens.length() > 0) {
-			if (tokens.length() >= 3) {
-				if (string_match(tokens[1], U"include")) {
-					if (tokens[2][0] == U'\"') {
-						String relativePath = string_unmangleQuote(tokens[2]);
-						String absolutePath = file_getTheoreticalAbsolutePath(relativePath, parentFolder, LOCAL_PATH_SYNTAX);
-						result.includes.pushConstruct(absolutePath, lineNumber);
-					}
-				}
-			}
-			tokens.clear();
-		}
-	});
-}
-
-void analyzeFromFile(ProjectContext &context, const ReadableString& absolutePath) {
-	if (findDependency(context, absolutePath) != -1) {
-		// Already analyzed the current entry. Abort to prevent duplicate dependencies.
-		return;
-	}
-	Extension extension = extensionFromString(file_getExtension(absolutePath));
-	if (extension != Extension::Unknown) {
-		// Create a new dependency for the file.
-		int64_t parentIndex = context.dependencies.length();
-		context.dependencies.push(Dependency(absolutePath, extension));
-		// Summarize the file's content.
-		analyzeFile(context.dependencies[parentIndex], absolutePath, extension);
-		// Continue analyzing recursively into the file's dependencies.
-		for (int64_t i = 0; i < context.dependencies[parentIndex].includes.length(); i++) {
-			analyzeFromFile(context, context.dependencies[parentIndex].includes[i].path);
-		}
-		for (int64_t l = 0; l < context.dependencies[parentIndex].links.length(); l++) {
-			analyzeFromFile(context, context.dependencies[parentIndex].links[l].path);
-		}
-	}
-}
-
-static void debugPrintDependencyList(const List<Connection> &connnections, const ReadableString verb) {
-	for (int64_t c = 0; c < connnections.length(); c++) {
-		int64_t lineNumber = connnections[c].lineNumber;
-		if (lineNumber != -1) {
-			printText(U"  @", lineNumber, U"\t");
-		} else {
-			printText(U"    \t");
-		}
-		printText(U" ", verb, U" ", file_getPathlessName(connnections[c].path), U"\n");
-	}
-}
-
-void printDependencies(ProjectContext &context) {
-	for (int64_t d = 0; d < context.dependencies.length(); d++) {
-		printText(U"* ", file_getPathlessName(context.dependencies[d].path), U"\n");
-		debugPrintDependencyList(context.dependencies[d].includes, U"including");
-		debugPrintDependencyList(context.dependencies[d].links, U"linking");
-	}
-}
-
-static void script_printMessage(String &output, ScriptLanguage language, const ReadableString message) {
-	if (language == ScriptLanguage::Batch) {
-		string_append(output, U"echo ", message, U"\n");
-	} else if (language == ScriptLanguage::Bash) {
-		string_append(output, U"echo ", message, U"\n");
-	}
-}
-
-static void traverserHeaderChecksums(ProjectContext &context, uint64_t &target, int64_t dependencyIndex) {
-	// Use checksums from headers
-	for (int64_t h = 0; h < context.dependencies[dependencyIndex].includes.length(); h++) {
-		int64_t includedIndex = context.dependencies[dependencyIndex].includes[h].dependencyIndex;
-		if (!context.dependencies[includedIndex].visited) {
-			//printText(U"	traverserHeaderChecksums(context, ", includedIndex, U") ", context.dependencies[includedIndex].path, "\n");
-			// Bitwise exclusive or is both order independent and entropy preserving for non-repeated content.
-			target = target ^ context.dependencies[includedIndex].contentChecksum;
-			// Just have to make sure that the same checksum is not used twice.
-			context.dependencies[includedIndex].visited = true;
-			// Use checksums from headers recursively
-			traverserHeaderChecksums(context, target, includedIndex);
-		}
-	}
-}
-
-static uint64_t getCombinedChecksum(ProjectContext &context, int64_t dependencyIndex) {
-	//printText(U"getCombinedChecksum(context, ", dependencyIndex, U") ", context.dependencies[dependencyIndex].path, "\n");
-	for (int64_t d = 0; d < context.dependencies.length(); d++) {
-		context.dependencies[d].visited = false;
-	}
-	context.dependencies[dependencyIndex].visited = true;
-	uint64_t result = context.dependencies[dependencyIndex].contentChecksum;
-	traverserHeaderChecksums(context, result, dependencyIndex);
-	return result;
-}
-
-static int64_t findObject(SessionContext &source, uint64_t identityChecksum) {
-	for (int64_t o = 0; o < source.sourceObjects.length(); o++) {
-		if (source.sourceObjects[o].identityChecksum == identityChecksum) {
-			return o;
-		}
-	}
-	return -1;
-}
-
-void gatherBuildInstructions(SessionContext &output, ProjectContext &context, Machine &settings, ReadableString programPath) {
-	// The compiler is often a global alias, so the user must supply either an alias or an absolute path.
-	ReadableString compilerName = getFlag(settings, U"Compiler", U"g++"); // Assume g++ as the compiler if not specified.
-	ReadableString compileFrom = getFlag(settings, U"CompileFrom", U"");
-	// Check if the build system was asked to run the compiler from a specific folder.
-	bool changePath = (string_length(compileFrom) > 0);
-	if (changePath) {
-		printText(U"Using ", compilerName, " as the compiler executed from ", compileFrom, ".\n");
-	} else {
-		printText(U"Using ", compilerName, " as the compiler from the current directory.\n");
-	}
-	// TODO: Warn if -DNDEBUG, -DDEBUG, or optimization levels are given directly.
-	//       Using the variables instead is both more flexible by accepting input arguments
-	//       and keeping the same format to better reuse compiled objects.
-	if (getFlagAsInteger(settings, U"Debug")) {
-		printText(U"Building with debug mode.\n");
-		settings.compilerFlags.push(U"-DDEBUG");
-	} else {
-		printText(U"Building with release mode.\n");
-		settings.compilerFlags.push(U"-DNDEBUG");
-	}
-	if (getFlagAsInteger(settings, U"StaticRuntime")) {
-		if (getFlagAsInteger(settings, U"Windows")) {
-			printText(U"Building with static runtime. Your application's binary will be bigger but can run without needing any installer.\n");
-			settings.compilerFlags.push(U"-static");
-			settings.compilerFlags.push(U"-static-libgcc");
-			settings.compilerFlags.push(U"-static-libstdc++");
-			settings.linkerFlags.push(U"-static");
-			settings.linkerFlags.push(U"-static-libgcc");
-			settings.linkerFlags.push(U"-static-libstdc++");
-		} else {
-			printText(U"The target platform does not support static linking of runtime. But don't worry about bundling any runtimes, because it comes with most of the Posix compliant operating systems.\n");
-		}
-	} else {
-		printText(U"Building with dynamic runtime. Don't forget to bundle the C and C++ runtimes for systems that don't have it pre-installed.\n");
-	}
-	ReadableString optimizationLevel = getFlag(settings, U"Optimization", U"2");
-		printText(U"Building with optimization level ", optimizationLevel, U".\n");
-	settings.compilerFlags.push(string_combine(U"-O", optimizationLevel));
-
-	// Convert lists of linker and compiler flags into strings.
-	// TODO: Give a warning if two contradictory flags are used, such as optimization levels and language versions.
-	// TODO: Make sure that no spaces are inside of the flags, because that can mess up detection of pre-existing and contradictory arguments.
-	// TODO: Use groups of compiler flags, so that they can be generated in the last step.
-	//       This would allow calling the compiler directly when given a folder path for temporary files instead of a script path.
-	String generatedCompilerFlags;
-	for (int64_t i = 0; i < settings.compilerFlags.length(); i++) {
-		string_append(generatedCompilerFlags, " ", settings.compilerFlags[i]);
-	}
-	String linkerFlags;
-	for (int64_t i = 0; i < settings.linkerFlags.length(); i++) {
-		string_append(linkerFlags, " -l", settings.linkerFlags[i]);
-	}
-	printText(U"Generating build instructions for ", programPath, U" using settings:\n");
-	printText(U"  Compiler flags:", generatedCompilerFlags, U"\n");
-	printText(U"  Linker flags:", linkerFlags, U"\n");
-	for (int64_t v = 0; v < settings.variables.length(); v++) {
-		printText(U"  * ", settings.variables[v].key, U" = ", settings.variables[v].value);
-		if (settings.variables[v].inherited) {
-			printText(U" (inherited input)");
-		}
-		printText(U"\n");
-	}
-	printText(U"Listing source files to compile in the current session.\n");
-	// The current project's global indices to objects shared between all projects being built during the session.
-	List<int64_t> sourceObjectIndices;
-	bool hasSourceCode = false;
-	for (int64_t d = 0; d < context.dependencies.length(); d++) {
-		Extension extension = context.dependencies[d].extension;
-		if (extension == Extension::C || extension == Extension::Cpp) {
-			// Dependency paths are already absolute from the recursive search.
-			String sourcePath = context.dependencies[d].path;
-			String identity = string_combine(sourcePath, generatedCompilerFlags);
-			uint64_t identityChecksum = checksum(identity);
-			int64_t previousIndex = findObject(output, identityChecksum);
-			if (previousIndex == -1) {
-				// Content checksums were created while scanning for source code, so now we just combine each source file's content checksum with all its headers to get the combined checksum.
-				// The combined checksum represents the state after all headers are included recursively and given as input for compilation unit generating an object.
-				uint64_t combinedChecksum = getCombinedChecksum(context, d);
-				String objectPath = file_combinePaths(output.tempPath, string_combine(U"dfpsr_", identityChecksum, U"_", combinedChecksum, U".o"));
-				sourceObjectIndices.push(output.sourceObjects.length());
-				output.sourceObjects.pushConstruct(identityChecksum, combinedChecksum, sourcePath, objectPath, generatedCompilerFlags, compilerName, compileFrom);
-			} else {
-				// Link to this pre-existing source file.
-				sourceObjectIndices.push(previousIndex);
-			}
-			hasSourceCode = true;
-		}
-	}
-	if (hasSourceCode) {
-		printText(U"Listing target executable ", programPath, " in the current session.\n");
-		bool executeResult = getFlagAsInteger(settings, U"Supressed") == 0;
-		output.linkerSteps.pushConstruct(compilerName, compileFrom, programPath, settings.linkerFlags, sourceObjectIndices, executeResult);
-	} else {
-		printText(U"Filed to find any source code to compile when building ", programPath, U".\n");
-	}
-}
-
 static ScriptLanguage identifyLanguage(const ReadableString &filename) {
 	String scriptExtension = string_upperCase(file_getExtension(filename));
 	if (string_match(scriptExtension, U"BAT")) {
@@ -363,7 +15,15 @@ static ScriptLanguage identifyLanguage(const ReadableString &filename) {
 	}
 }
 
-void setCompilationFolder(String &generatedCode, ScriptLanguage language, String &currentPath, const ReadableString &newPath) {
+static void script_printMessage(String &output, ScriptLanguage language, const ReadableString message) {
+	if (language == ScriptLanguage::Batch) {
+		string_append(output, U"echo ", message, U"\n");
+	} else if (language == ScriptLanguage::Bash) {
+		string_append(output, U"echo ", message, U"\n");
+	}
+}
+
+static void setCompilationFolder(String &generatedCode, ScriptLanguage language, String &currentPath, const ReadableString &newPath) {
 	if (!string_match(currentPath, newPath)) {
 		if (string_length(currentPath) > 0) {
 			if (language == ScriptLanguage::Batch) {

+ 1 - 11
Source/tools/builder/generator.h

@@ -3,20 +3,10 @@
 #define DSR_BUILDER_GENERATOR_MODULE
 
 #include "../../DFPSR/api/fileAPI.h"
-#include "Machine.h"
+#include "builderTypes.h"
 
 using namespace dsr;
 
-// Analyze using calls from the machine
-void analyzeFromFile(ProjectContext &context, const ReadableString& entryPath);
-// Call from main when done analyzing source files
-void resolveDependencies(ProjectContext &context);
-
-// Visualize
-void printDependencies(ProjectContext &context);
-
-// Generate
-void gatherBuildInstructions(SessionContext &output, ProjectContext &context, Machine &settings, ReadableString programPath);
 void generateCompilationScript(SessionContext &output, const ReadableString &scriptPath);
 
 #endif

+ 1 - 0
Source/tools/builder/main.cpp

@@ -68,6 +68,7 @@ Project files:
 #include "../../DFPSR/api/fileAPI.h"
 #include "Machine.h"
 #include "expression.h"
+#include "analyzer.h"
 #include "generator.h"
 
 using namespace dsr;