Using 64-bit integers for indices.

Source/tools/builder/Machine.cpp

@@ -92,7 +92,7 @@ static void crawlSource(ProjectContext &context, const dsr::ReadableString &abso
 
 // Copy inherited variables from parent to child.
 static void inheritMachine(Machine &child, const Machine &parent) {
-	for (int v = 0; v < parent.variables.length(); v++) {
+	for (int64_t v = 0; v < parent.variables.length(); v++) {
 		String key = string_upperCase(parent.variables[v].key);
 		if (parent.variables[v].inherited) {
 			child.variables.push(parent.variables[v]);
@@ -105,7 +105,7 @@ static void interpretLine(SessionContext &output, ProjectContext &context, Machi
 		bool activeLine = target.activeStackDepth >= target.currentStackDepth;
 		/*
 		printText(activeLine ? U"interpret:" : U"ignore:");
-		for (int t = 0; t < tokens.length(); t++) {
+		for (int64_t t = 0; t < tokens.length(); t++) {
 			printText(U" [", tokens[t], U"]");
 		}
 		printText(U"\n");
@@ -194,7 +194,7 @@ void evaluateScript(SessionContext &output, ProjectContext &context, Machine &ta
 	List<String> currentLine; // Keep it fast and simple by only remembering tokens for the current line.
 	bool quoted = false;
 	bool commented = false;
-	for (int i = 0; i <= string_length(projectContent); i++) {
+	for (int64_t i = 0; i <= string_length(projectContent); i++) {
 		DsrChar c = projectContent[i];
 		// The null terminator does not really exist in projectContent,
 		//   but dsr::String returns a null character safely when requesting a character out of bound,
@@ -242,7 +242,7 @@ void buildProject(SessionContext &output, const ReadableString &projectFilePath,
 	printText("Building project at ", projectFilePath, "\n");
 	// Check if this project has begun building previously during this session.
 	String absolutePath = file_getAbsolutePath(projectFilePath);
-	for (int p = 0; p < initializedProjects.length(); p++) {
+	for (int64_t p = 0; p < initializedProjects.length(); p++) {
 		if (string_caseInsensitiveMatch(absolutePath, initializedProjects[p])) {
 			throwError(U"Found duplicate requests to build from the same initial script ", absolutePath, U" which could cause non-determinism if different arguments are given to each!\n");
 			return;
@@ -311,7 +311,7 @@ void build(SessionContext &output, const ReadableString &projectPath, Machine &s
 }
 
 void argumentsToSettings(Machine &settings, const List<String> &arguments, int64_t firstArgument) {
-	for (int a = firstArgument; a < arguments.length(); a++) {
+	for (int64_t a = firstArgument; a < arguments.length(); a++) {
 		String argument = arguments[a];
 		int64_t assignmentIndex = string_findFirst(argument, U'=');
 		if (assignmentIndex == -1) {

Source/tools/builder/expression.cpp

@@ -26,7 +26,7 @@ Precedence::Precedence(Notation notation, Associativity associativity)
 Symbol::Symbol(const dsr::ReadableString &token, bool atomic, int32_t depthOffset)
 : token(token), atomic(atomic), depthOffset(depthOffset) {}
 
-ReadableString expression_getToken(const List<String> &tokens, int index) {
+ReadableString expression_getToken(const List<String> &tokens, int64_t index) {
 	if (0 <= index && index < tokens.length()) {
 		return tokens[index];
 	} else {
@@ -184,7 +184,7 @@ struct TokenInfo {
 /*
 static String debugTokens(const List<TokenInfo> &info, int64_t infoStart, const List<String> &tokens, int64_t startTokenIndex, int64_t endTokenIndex) {
 	String result;
-	for (int t = startTokenIndex; t <= endTokenIndex; t++) {
+	for (int64_t t = startTokenIndex; t <= endTokenIndex; t++) {
 		int64_t infoIndex = t - infoStart;
 		if (t > startTokenIndex) {
 			string_appendChar(result, U' ');
@@ -192,7 +192,7 @@ static String debugTokens(const List<TokenInfo> &info, int64_t infoStart, const
 		string_append(result, tokens[t]);
 	}
 	string_append(result, U" : ");
-	for (int t = startTokenIndex; t <= endTokenIndex; t++) {
+	for (int64_t t = startTokenIndex; t <= endTokenIndex; t++) {
 		int64_t infoIndex = t - infoStart;
 		if (t > startTokenIndex) {
 			string_appendChar(result, U' ');
@@ -204,7 +204,7 @@ static String debugTokens(const List<TokenInfo> &info, int64_t infoStart, const
 */
 
 static int16_t identifySymbol(const ReadableString &token, const ExpressionSyntax &syntax) {
-	for (int s = 0; s < syntax.symbols.length(); s++) {
+	for (int64_t s = 0; s < syntax.symbols.length(); s++) {
 		if (syntax.symbols[s].atomic) {
 			if (string_match(token, syntax.symbols[s].token)) {
 				return s;
@@ -391,7 +391,7 @@ inline List<String> combineTokens(ARGS... args) {
 	return result;
 }
 
-static void expectResult(int &errorCount, const ReadableString &result, const ReadableString &expected) {
+static void expectResult(int64_t &errorCount, const ReadableString &result, const ReadableString &expected) {
 	if (string_match(result, expected)) {
 		printText(U"* Passed ", expected, U"\n");
 	} else {
@@ -412,12 +412,12 @@ void expression_runRegressionTests() {
 			return U"<ERROR:Unresolved identifier>";
 		}
 	};
-	/*for (int s = 0; s < defaultSyntax.symbols.length(); s++) {
+	/*for (int64_t s = 0; s < defaultSyntax.symbols.length(); s++) {
 		printText(U"Symbol ", defaultSyntax.symbols[s].token, U"\n");
 		if (validLeftmostToken(s, defaultSyntax)) printText(U"  Can be leftmost\n");
 		if (validRightmostToken(s, defaultSyntax)) printText(U"  Can be rightmost\n");
 	}*/
-	int ec = 0;
+	int64_t ec = 0;
 	expectResult(ec, expression_evaluate(combineTokens(U""), context), U"<ERROR:Unresolved identifier>");
 	expectResult(ec, expression_evaluate(combineTokens(U"0"), context), U"0");
 	expectResult(ec, expression_evaluate(combineTokens(U"(", U"19", U")"), context), U"19");

Source/tools/builder/expression.h

@@ -56,7 +56,7 @@ struct ExpressionSyntax {
 
 dsr::String expression_unwrapIfNeeded(const dsr::ReadableString &text);
 
-dsr::ReadableString expression_getToken(const dsr::List<dsr::String> &tokens, int index);
+dsr::ReadableString expression_getToken(const dsr::List<dsr::String> &tokens, int64_t index);
 
 int64_t expression_interpretAsInteger(const dsr::ReadableString &value);
 

Source/tools/builder/generator.cpp

@@ -8,7 +8,7 @@ static uint64_t checksum(const ReadableString& text) {
 	uint64_t b = 0xF42B1583;
 	uint64_t c = 0xA6815E74;
 	uint64_t d = 0;
-	for (int i = 0; i < string_length(text); i++) {
+	for (int64_t i = 0; i < string_length(text); i++) {
 		a = (b * c + ((i * 3756 + 2654) & 58043)) & 0xFFFFFFFF;
 		b = (231 + text[i] * (a & 154) + c * 867 + 28294061) & 0xFFFFFFFF;
 		c = (a ^ b ^ (text[i] * 1543217521)) & 0xFFFFFFFF;
@@ -23,7 +23,7 @@ static uint64_t checksum(const Buffer& buffer) {
 	uint64_t b = 0xF42B1583;
 	uint64_t c = 0xA6815E74;
 	uint64_t d = 0;
-	for (int i = 0; i < buffer_getSize(buffer); i++) {
+	for (int64_t i = 0; i < buffer_getSize(buffer); i++) {
 		a = (b * c + ((i * 3756 + 2654) & 58043)) & 0xFFFFFFFF;
 		b = (231 + data[i] * (a & 154) + c * 867 + 28294061) & 0xFFFFFFFF;
 		c = (a ^ b ^ (data[i] * 1543217521)) & 0xFFFFFFFF;
@@ -42,7 +42,7 @@ static void interpretPreprocessing(ProjectContext &context, int64_t parentIndex,
 static void analyzeCode(ProjectContext &context, int64_t parentIndex, String content, const ReadableString &parentFolder);
 
 static int64_t findDependency(ProjectContext &context, const ReadableString& findPath) {
-	for (int d = 0; d < context.dependencies.length(); d++) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
 		if (string_match(context.dependencies[d].path, findPath)) {
 			return d;
 		}
@@ -55,16 +55,16 @@ static void resolveConnection(ProjectContext &context, Connection &connection) {
 }
 
 static void resolveDependency(ProjectContext &context, Dependency &dependency) {
-	for (int l = 0; l < dependency.links.length(); l++) {
+	for (int64_t l = 0; l < dependency.links.length(); l++) {
 		resolveConnection(context, dependency.links[l]);
 	}
-	for (int i = 0; i < dependency.includes.length(); i++) {
+	for (int64_t i = 0; i < dependency.includes.length(); i++) {
 		resolveConnection(context, dependency.includes[i]);
 	}
 }
 
 void resolveDependencies(ProjectContext &context) {
-	for (int d = 0; d < context.dependencies.length(); d++) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
 		resolveDependency(context, context.dependencies[d]);
 	}
 }
@@ -92,7 +92,7 @@ static void flushToken(List<String> &target, String &currentToken) {
 
 static void tokenize(List<String> &target, const ReadableString& line) {
 	String currentToken;
-	for (int i = 0; i < string_length(line); i++) {
+	for (int64_t i = 0; i < string_length(line); i++) {
 		DsrChar c = line[i];
 		DsrChar nextC = line[i + 1];
 		if (c == U'#' && nextC == U'#') {
@@ -151,7 +151,7 @@ void analyzeFromFile(ProjectContext &context, const ReadableString& absolutePath
 		// Already analyzed the current entry. Abort to prevent duplicate dependencies.
 		return;
 	}
-	int lastDotIndex = string_findLast(absolutePath, U'.');
+	int64_t lastDotIndex = string_findLast(absolutePath, U'.');
 	if (lastDotIndex != -1) {
 		Extension extension = extensionFromString(string_after(absolutePath, lastDotIndex));
 		if (extension != Extension::Unknown) {
@@ -180,7 +180,7 @@ void analyzeFromFile(ProjectContext &context, const ReadableString& absolutePath
 }
 
 static void debugPrintDependencyList(const List<Connection> &connnections, const ReadableString verb) {
-	for (int c = 0; c < connnections.length(); c++) {
+	for (int64_t c = 0; c < connnections.length(); c++) {
 		int64_t lineNumber = connnections[c].lineNumber;
 		if (lineNumber != -1) {
 			printText(U"  @", lineNumber, U"\t");
@@ -192,7 +192,7 @@ static void debugPrintDependencyList(const List<Connection> &connnections, const
 }
 
 void printDependencies(ProjectContext &context) {
-	for (int d = 0; d < context.dependencies.length(); d++) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
 		printText(U"* ", file_getPathlessName(context.dependencies[d].path), U"\n");
 		debugPrintDependencyList(context.dependencies[d].includes, U"including");
 		debugPrintDependencyList(context.dependencies[d].links, U"linking");
@@ -209,7 +209,7 @@ static void script_printMessage(String &output, ScriptLanguage language, const R
 
 static void traverserHeaderChecksums(ProjectContext &context, uint64_t &target, int64_t dependencyIndex) {
 	// Use checksums from headers
-	for (int h = 0; h < context.dependencies[dependencyIndex].includes.length(); h++) {
+	for (int64_t h = 0; h < context.dependencies[dependencyIndex].includes.length(); h++) {
 		int64_t includedIndex = context.dependencies[dependencyIndex].includes[h].dependencyIndex;
 		if (!context.dependencies[includedIndex].visited) {
 			//printText(U"	traverserHeaderChecksums(context, ", includedIndex, U") ", context.dependencies[includedIndex].path, "\n");
@@ -225,7 +225,7 @@ static void traverserHeaderChecksums(ProjectContext &context, uint64_t &target,
 
 static uint64_t getCombinedChecksum(ProjectContext &context, int64_t dependencyIndex) {
 	//printText(U"getCombinedChecksum(context, ", dependencyIndex, U") ", context.dependencies[dependencyIndex].path, "\n");
-	for (int d = 0; d < context.dependencies.length(); d++) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
 		context.dependencies[d].visited = false;
 	}
 	context.dependencies[dependencyIndex].visited = true;
@@ -289,17 +289,17 @@ void gatherBuildInstructions(SessionContext &output, ProjectContext &context, Ma
 	// TODO: Use groups of compiler flags, so that they can be generated in the last step.
 	//       This would allow calling the compiler directly when given a folder path for temporary files instead of a script path.
 	String generatedCompilerFlags;
-	for (int i = 0; i < settings.compilerFlags.length(); i++) {
+	for (int64_t i = 0; i < settings.compilerFlags.length(); i++) {
 		string_append(generatedCompilerFlags, " ", settings.compilerFlags[i]);
 	}
 	String linkerFlags;
-	for (int i = 0; i < settings.linkerFlags.length(); i++) {
+	for (int64_t i = 0; i < settings.linkerFlags.length(); i++) {
 		string_append(linkerFlags, " -l", settings.linkerFlags[i]);
 	}
 	printText(U"Generating build instructions for ", programPath, U" using settings:\n");
 	printText(U"  Compiler flags:", generatedCompilerFlags, U"\n");
 	printText(U"  Linker flags:", linkerFlags, U"\n");
-	for (int v = 0; v < settings.variables.length(); v++) {
+	for (int64_t v = 0; v < settings.variables.length(); v++) {
 		printText(U"  * ", settings.variables[v].key, U" = ", settings.variables[v].value);
 		if (settings.variables[v].inherited) {
 			printText(U" (inherited input)");
@@ -310,7 +310,7 @@ void gatherBuildInstructions(SessionContext &output, ProjectContext &context, Ma
 	// The current project's global indices to objects shared between all projects being built during the session.
 	List<int64_t> sourceObjectIndices;
 	bool hasSourceCode = false;
-	for (int d = 0; d < context.dependencies.length(); d++) {
+	for (int64_t d = 0; d < context.dependencies.length(); d++) {
 		Extension extension = context.dependencies[d].extension;
 		if (extension == Extension::C || extension == Extension::Cpp) {
 			// Dependency paths are already absolute from the recursive search.
@@ -387,7 +387,7 @@ void generateCompilationScript(SessionContext &input, const ReadableString &scri
 
 	// Generate code for compiling source code into objects.
 	printText(U"Generating code for compiling ", input.sourceObjects.length(), U" objects.\n");
-	for (int o = 0; o < input.sourceObjects.length(); o++) {
+	for (int64_t o = 0; o < input.sourceObjects.length(); o++) {
 		SourceObject *sourceObject = &(input.sourceObjects[o]);
 		printText(U"\t* ", sourceObject->sourcePath, U"\n");
 		setCompilationFolder(generatedCode, language, currentPath, sourceObject->compileFrom);
@@ -414,20 +414,20 @@ void generateCompilationScript(SessionContext &input, const ReadableString &scri
 
 	// Generate code for linking objects into executables.
 	printText(U"Generating code for linking ", input.linkerSteps.length(), U" executables:\n");
-	for (int l = 0; l < input.linkerSteps.length(); l++) {
+	for (int64_t l = 0; l < input.linkerSteps.length(); l++) {
 		LinkingStep *linkingStep = &(input.linkerSteps[l]);
 		String programPath = linkingStep->binaryName;
 		printText(U"\tGenerating code for linking ", programPath, U" of :\n");
 		setCompilationFolder(generatedCode, language, currentPath, linkingStep->compileFrom);
 		String linkerFlags;
-		for (int lib = 0; lib < linkingStep->linkerFlags.length(); lib++) {
+		for (int64_t lib = 0; lib < linkingStep->linkerFlags.length(); lib++) {
 			String library = linkingStep->linkerFlags[lib];
 			string_append(linkerFlags, " -l", library);
 			printText(U"\t\t* ", library, U" library\n");
 		}
 		// Generate a list of object paths from indices.
 		String allObjects;
-		for (int i = 0; i < linkingStep->sourceObjectIndices.length(); i++) {
+		for (int64_t i = 0; i < linkingStep->sourceObjectIndices.length(); i++) {
 			int64_t objectIndex = linkingStep->sourceObjectIndices[i];
 			SourceObject *sourceObject = &(input.sourceObjects[objectIndex]);
 			if (objectIndex >= 0 || objectIndex < input.sourceObjects.length()) {

Source/tools/builder/main.cpp

@@ -6,6 +6,15 @@
 // TODO:
 //  * Let ProjectContext refer to a global dependency tree using indices instead of having its own List of Dependency (dependencies).
 //    This would greatly improve performance by not scanning for includes and taking content checksums multiple times for the same files.
+//  * Create a file with aliases, so that import can use
+//      Import <DFPSR>
+//    instead of
+//      Import "../../DFPSR/DFPSR.DsrHead"
+//  * Call the compiler directly when the temp folder is given without any script name.
+//    Use it to run multiple instances of the compiler at the same time on different CPU cores.
+//  * Add a flag for enabling Allocator.cpp.
+//    The macro for supressing it will then no longer be needed when using the new build system.
+//  * Improve entropy in checksums using a more advanced algorithm to reduce the risk of conflicts.
 //  * Implement more features for the machine, such as:
 //    * else and elseif cases.
 //    * Temporarily letting the theoretical path go into another folder within a scope, similar to if statements but only affecting the path.