Minimize TokenPos size by using `i32` for line/column/offset and file_id instead of `String`

To make `i32` safe, the parser limits the file size of odin files to a maximum of 2GiB (which will be good enough for the vast vast majority of cases)

src/check_decl.cpp

@@ -799,14 +799,14 @@ void check_proc_decl(CheckerContext *ctx, Entity *e, DeclInfo *d) {
 				if (!are_signatures_similar_enough(this_type, other_type)) {
 					error(d->proc_lit,
 					      "Redeclaration of foreign procedure '%.*s' with different type signatures\n"
-					      "\tat %.*s(%td:%td)",
-					      LIT(name), LIT(pos.file), pos.line, pos.column);
+					      "\tat %s",
+					      LIT(name), token_pos_to_string(pos));
 				}
 			} else if (!are_types_identical(this_type, other_type)) {
 				error(d->proc_lit,
 				      "Foreign entity '%.*s' previously declared elsewhere with a different type\n"
-				      "\tat %.*s(%td:%td)",
-				      LIT(name), LIT(pos.file), pos.line, pos.column);
+				      "\tat %s",
+				      LIT(name), token_pos_to_string(pos));
 			}
 		} else if (name == "main") {
 			error(d->proc_lit, "The link name 'main' is reserved for internal use");
@@ -828,8 +828,8 @@ void check_proc_decl(CheckerContext *ctx, Entity *e, DeclInfo *d) {
 				// TODO(bill): Better error message?
 				error(d->proc_lit,
 				      "Non unique linking name for procedure '%.*s'\n"
-				      "\tother at %.*s(%td:%td)",
-				      LIT(name), LIT(pos.file), pos.line, pos.column);
+				      "\tother at %s",
+				      LIT(name), token_pos_to_string(pos));
 			} else if (name == "main") {
 				error(d->proc_lit, "The link name 'main' is reserved for internal use");
 			} else {
@@ -919,8 +919,8 @@ void check_global_variable_decl(CheckerContext *ctx, Entity *e, Ast *type_expr,
 			if (!are_types_identical(this_type, other_type)) {
 				error(e->token,
 				      "Foreign entity '%.*s' previously declared elsewhere with a different type\n"
-				      "\tat %.*s(%td:%td)",
-				      LIT(name), LIT(pos.file), pos.line, pos.column);
+				      "\tat %s",
+				      LIT(name), token_pos_to_string(pos));
 			}
 		} else {
 			string_map_set(fp, key, e);
@@ -1059,7 +1059,7 @@ void check_proc_group_decl(CheckerContext *ctx, Entity *pg_entity, DeclInfo *d)
 			}
 
 			if (is_invalid) {
-				error_line("\tprevious procedure at %.*s(%td:%td)\n", LIT(pos.file), pos.line, pos.column);
+				error_line("\tprevious procedure at %s\n", token_pos_to_string(pos));
 				q->type = t_invalid;
 			}
 		}

src/check_expr.cpp

@@ -3955,7 +3955,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32
 			gbAllocator a = heap_allocator();
 
 			GB_ASSERT(o.value.kind == ExactValue_String);
-			String base_dir = dir_from_path(bd->token.pos.file);
+			String base_dir = dir_from_path(get_file_path_string(bd->token.pos.file_id));
 			String original_string = o.value.value_string;
 
 
@@ -7239,7 +7239,7 @@ CallArgumentData check_call_arguments(CheckerContext *c, Operand *operand, Type
 				if (proc->kind == Entity_Variable) {
 					sep = ":=";
 				}
-				error_line("\t%.*s%.*s%.*s %s %s at %.*s(%td:%td)\n", LIT(prefix), LIT(prefix_sep), LIT(name), sep, pt, LIT(pos.file), pos.line, pos.column);
+				error_line("\t%.*s%.*s%.*s %s %s at %s\n", LIT(prefix), LIT(prefix_sep), LIT(name), sep, pt, token_pos_to_string(pos));
 			}
 			if (procs.count > 0) {
 				error_line("\n");
@@ -7300,8 +7300,7 @@ CallArgumentData check_call_arguments(CheckerContext *c, Operand *operand, Type
 						error_line("\n\t");
 					}
 				}
-				error_line("at %.*s(%td:%td)\n", LIT(pos.file), pos.line, pos.column);
-				// error_line("\t%.*s %s %s at %.*s(%td:%td) %lld\n", LIT(name), sep, pt, LIT(pos.file), pos.line, pos.column, valids[i].score);
+				error_line("at %s\n", token_pos_to_string(pos));
 			}
 			result_type = t_invalid;
 		} else {
@@ -8255,7 +8254,7 @@ ExprKind check_expr_base_internal(CheckerContext *c, Operand *o, Ast *node, Type
 		o->mode = Addressing_Constant;
 		if (bd->name == "file") {
 			o->type = t_untyped_string;
-			o->value = exact_value_string(bd->token.pos.file);
+			o->value = exact_value_string(get_file_path_string(bd->token.pos.file_id));
 		} else if (bd->name == "line") {
 			o->type = t_untyped_integer;
 			o->value = exact_value_i64(bd->token.pos.line);

src/check_stmt.cpp

@@ -540,11 +540,11 @@ bool check_using_stmt_entity(CheckerContext *ctx, AstUsingStmt *us, Ast *expr, b
 				gbString expr_str = expr_to_string(expr);
 				error(us->token,
 				      "Namespace collision while 'using' '%s' of: %.*s\n"
-				      "\tat %.*s(%td:%td)\n"
-				      "\tat %.*s(%td:%td)",
+				      "\tat %s\n"
+				      "\tat %s",
 				      expr_str, LIT(found->token.string),
-				      LIT(found->token.pos.file), found->token.pos.line, found->token.pos.column,
-				      LIT(decl->token.pos.file), decl->token.pos.line, decl->token.pos.column
+				      token_pos_to_string(found->token.pos),
+				      token_pos_to_string(decl->token.pos)
 				      );
 				gb_string_free(expr_str);
 				return false;
@@ -644,14 +644,12 @@ void add_constant_switch_case(CheckerContext *ctx, Map<TypeAndToken> *seen, Oper
 				gbString expr_str = expr_to_string(operand.expr);
 				error(operand.expr,
 				      "Duplicate case '%s'\n"
-				      "\tprevious case at %.*s(%td:%td)",
+				      "\tprevious case at %s",
 				      expr_str,
-				      LIT(pos.file), pos.line, pos.column);
+				      token_pos_to_string(pos));
 				gb_string_free(expr_str);
 			} else {
-				error(operand.expr,
-				      "Duplicate case found with previous case at %.*s(%td:%td)",
-				      LIT(pos.file), pos.line, pos.column);
+				error(operand.expr, "Duplicate case found with previous case at %s", token_pos_to_string(pos));
 			}
 			return;
 		}
@@ -768,8 +766,7 @@ void check_inline_range_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) {
 				TokenPos pos = found->token.pos;
 				error(token,
 				      "Redeclaration of '%.*s' in this scope\n"
-				      "\tat %.*s(%td:%td)",
-				      LIT(str), LIT(pos.file), pos.line, pos.column);
+				      "\tat %s", LIT(str), token_pos_to_string(pos));
 				entity = found;
 			}
 		} else {
@@ -871,8 +868,7 @@ void check_switch_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) {
 				TokenPos pos = ast_token(first_default).pos;
 				error(stmt,
 				           "multiple default clauses\n"
-				           "\tfirst at %.*s(%td:%td)",
-				           LIT(pos.file), pos.line, pos.column);
+				           "\tfirst at %s", token_pos_to_string(pos));
 			} else {
 				first_default = default_stmt;
 			}
@@ -1134,8 +1130,7 @@ void check_type_switch_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) {
 				TokenPos pos = ast_token(first_default).pos;
 				error(stmt,
 				      "Multiple default clauses\n"
-				      "\tfirst at %.*s(%td:%td)",
-				      LIT(pos.file), pos.line, pos.column);
+				      "\tfirst at %s", token_pos_to_string(pos));
 			} else {
 				first_default = default_stmt;
 			}
@@ -1205,9 +1200,9 @@ void check_type_switch_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) {
 					gbString expr_str = expr_to_string(y.expr);
 					error(y.expr,
 					           "Duplicate type case '%s'\n"
-					           "\tprevious type case at %.*s(%td:%td)",
+					           "\tprevious type case at %s",
 					           expr_str,
-					           LIT(pos.file), pos.line, pos.column);
+					           token_pos_to_string(pos));
 					gb_string_free(expr_str);
 					break;
 				}
@@ -1840,8 +1835,8 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) {
 					TokenPos pos = found->token.pos;
 					error(token,
 					      "Redeclaration of '%.*s' in this scope\n"
-					      "\tat %.*s(%td:%td)",
-					      LIT(str), LIT(pos.file), pos.line, pos.column);
+					      "\tat %s",
+					      LIT(str), token_pos_to_string(pos));
 					entity = found;
 				}
 			} else {
@@ -2055,8 +2050,8 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) {
 						TokenPos pos = found->token.pos;
 						error(token,
 						      "Redeclaration of '%.*s' in this scope\n"
-						      "\tat %.*s(%td:%td)",
-						      LIT(str), LIT(pos.file), pos.line, pos.column);
+						      "\tat %s",
+						      LIT(str), token_pos_to_string(pos));
 						entity = found;
 					}
 				}
@@ -2166,8 +2161,8 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) {
 						if (!are_types_identical(this_type, other_type)) {
 							error(e->token,
 							      "Foreign entity '%.*s' previously declared elsewhere with a different type\n"
-							      "\tat %.*s(%td:%td)",
-							      LIT(name), LIT(pos.file), pos.line, pos.column);
+							      "\tat %s",
+							      LIT(name), token_pos_to_string(pos));
 						}
 					} else {
 						string_map_set(fp, key, e);

src/checker.cpp

@@ -507,7 +507,7 @@ bool check_vet_shadowing(Checker *c, Entity *e, VettedEntity *ve) {
 	}
 
 	// NOTE(bill): The entities must be in the same file
-	if (e->token.pos.file != shadowed->token.pos.file) {
+	if (e->token.pos.file_id != shadowed->token.pos.file_id) {
 		return false;
 	}
 	// NOTE(bill): The shaded identifier must appear before this one to be an
@@ -1127,15 +1127,15 @@ bool redeclaration_error(String name, Entity *prev, Entity *found) {
 		if (found->flags & EntityFlag_Result) {
 			error(prev->token,
 			      "Direct shadowing of the named return value '%.*s' in this scope through 'using'\n"
-			      "\tat %.*s(%td:%td)",
+			      "\tat %s",
 			      LIT(name),
-			      LIT(up->token.pos.file), up->token.pos.line, up->token.pos.column);
+			      token_pos_to_string(up->token.pos));
 		} else {
 			error(prev->token,
 			      "Redeclaration of '%.*s' in this scope through 'using'\n"
-			      "\tat %.*s(%td:%td)",
+			      "\tat %s",
 			      LIT(name),
-			      LIT(up->token.pos.file), up->token.pos.line, up->token.pos.column);
+			      token_pos_to_string(up->token.pos));
 		}
 	} else {
 		if (pos == prev->token.pos) {
@@ -1145,15 +1145,15 @@ bool redeclaration_error(String name, Entity *prev, Entity *found) {
 		if (found->flags & EntityFlag_Result) {
 			error(prev->token,
 			      "Direct shadowing of the named return value '%.*s' in this scope\n"
-			      "\tat %.*s(%td:%td)",
+			      "\tat %s",
 			      LIT(name),
-			      LIT(pos.file), pos.line, pos.column);
+			      token_pos_to_string(pos));
 		} else {
 			error(prev->token,
 			      "Redeclaration of '%.*s' in this scope\n"
-			      "\tat %.*s(%td:%td)",
+			      "\tat %s",
 			      LIT(name),
-			      LIT(pos.file), pos.line, pos.column);
+			      token_pos_to_string(pos));
 		}
 	}
 	return false;
@@ -3524,7 +3524,7 @@ void add_import_dependency_node(Checker *c, Ast *decl, Map<ImportGraphNode *> *M
 				gb_printf_err("%.*s\n", LIT(pkg->fullpath));
 			}
 			Token token = ast_token(decl);
-			gb_printf_err("%.*s(%td:%td)\n", LIT(token.pos.file), token.pos.line, token.pos.column);
+			gb_printf_err("%s\n", token_pos_to_string(token.pos));
 			GB_PANIC("Unable to find package: %.*s", LIT(path));
 		}
 		AstPackage *pkg = *found;
@@ -3699,7 +3699,7 @@ void check_add_import_decl(CheckerContext *ctx, Ast *decl) {
 				AstPackage *pkg = pkgs->entries[pkg_index].value;
 				gb_printf_err("%.*s\n", LIT(pkg->fullpath));
 			}
-			gb_printf_err("%.*s(%td:%td)\n", LIT(token.pos.file), token.pos.line, token.pos.column);
+			gb_printf_err("%s\n", token_pos_to_string(token.pos));
 			GB_PANIC("Unable to find scope for package: %.*s", LIT(id->fullpath));
 		} else {
 			AstPackage *pkg = *found;
@@ -4714,9 +4714,9 @@ void check_parsed_files(Checker *c) {
 		Entity *e = scope_lookup_current(s, str_lit("main"));
 		if (e == nullptr) {
 			Token token = {};
-			token.pos.file   = s->pkg->fullpath;
-			token.pos.line   = 1;
-			token.pos.column = 1;
+			token.pos.file_id = 0;
+			token.pos.line    = 1;
+			token.pos.column  = 1;
 			if (s->pkg->files.count > 0) {
 				AstFile *f = s->pkg->files[0];
 				if (f->tokens.count > 0) {

src/ir.cpp

@@ -2510,7 +2510,7 @@ irDebugInfo *ir_add_debug_info_type(irModule *module, Type *type, Entity *e, irD
 		e = type->Named.type_name;
 		if (e) {
 			CheckerInfo *info = module->info;
-			file = ir_add_debug_info_file(module, ast_file_of_filename(info, e->token.pos.file));
+			file = ir_add_debug_info_file(module, ast_file_of_filename(info, get_file_path_string(e->token.pos.file_id)));
 			// TODO(lachsinc): Determine proper scope for type declaration location stuff.
 			scope = file;
 		}
@@ -2875,7 +2875,7 @@ irDebugInfo *ir_add_debug_info_global(irModule *module, irValue *v) {
 
 	// Create or fetch file debug info.
 	CheckerInfo *info = module->info;
-	String filename = e->token.pos.file;
+	String filename = get_file_path_string(e->token.pos.file_id);
 	AstFile *f = ast_file_of_filename(info, filename);
 	GB_ASSERT_NOT_NULL(f);
 	irDebugInfo *scope = ir_add_debug_info_file(module, f);
@@ -2964,7 +2964,7 @@ irDebugInfo *ir_add_debug_info_proc(irProcedure *proc) {
 
 	// Add / retrieve debug info for file.
 	CheckerInfo *info = proc->module->info;
-	String filename = proc->entity->token.pos.file;
+	String filename = get_file_path_string(proc->entity->token.pos.file_id);
 	AstFile *f = ast_file_of_filename(info, filename);
 	irDebugInfo *file = nullptr;
 	if (f) {
@@ -6419,7 +6419,7 @@ irValue *ir_emit_union_cast(irProcedure *proc, irValue *value, Type *type, Token
 			auto args = array_make<irValue *>(ir_allocator(), 6);
 			args[0] = ok;
 
-			args[1] = ir_find_or_add_entity_string(proc->module, pos.file);
+			args[1] = ir_find_or_add_entity_string(proc->module, get_file_path_string(pos.file_id));
 			args[2] = ir_const_int(pos.line);
 			args[3] = ir_const_int(pos.column);
 
@@ -6479,7 +6479,7 @@ irAddr ir_emit_any_cast_addr(irProcedure *proc, irValue *value, Type *type, Toke
 		auto args = array_make<irValue *>(ir_allocator(), 6);
 		args[0] = ok;
 
-		args[1] = ir_find_or_add_entity_string(proc->module, pos.file);
+		args[1] = ir_find_or_add_entity_string(proc->module, get_file_path_string(pos.file_id));
 		args[2] = ir_const_int(pos.line);
 		args[3] = ir_const_int(pos.column);
 
@@ -6676,7 +6676,7 @@ void ir_emit_bounds_check(irProcedure *proc, Token token, irValue *index, irValu
 	len = ir_emit_conv(proc, len, t_int);
 
 	gbAllocator a = ir_allocator();
-	irValue *file = ir_find_or_add_entity_string(proc->module, token.pos.file);
+	irValue *file = ir_find_or_add_entity_string(proc->module, get_file_path_string(token.pos.file_id));
 	irValue *line = ir_const_int(token.pos.line);
 	irValue *column = ir_const_int(token.pos.column);
 
@@ -6700,7 +6700,7 @@ void ir_emit_slice_bounds_check(irProcedure *proc, Token token, irValue *low, ir
 	}
 
 	gbAllocator a = ir_allocator();
-	irValue *file = ir_find_or_add_entity_string(proc->module, token.pos.file);
+	irValue *file = ir_find_or_add_entity_string(proc->module, get_file_path_string(token.pos.file_id));
 	irValue *line = ir_const_int(token.pos.line);
 	irValue *column = ir_const_int(token.pos.column);
 	high = ir_emit_conv(proc, high, t_int);
@@ -6739,7 +6739,7 @@ void ir_emit_dynamic_array_bounds_check(irProcedure *proc, Token token, irValue
 	}
 
 	gbAllocator a = ir_allocator();
-	irValue *file = ir_find_or_add_entity_string(proc->module, token.pos.file);
+	irValue *file = ir_find_or_add_entity_string(proc->module, get_file_path_string(token.pos.file_id));
 	irValue *line = ir_const_int(token.pos.line);
 	irValue *column = ir_const_int(token.pos.column);
 	low  = ir_emit_conv(proc, low,  t_int);
@@ -7077,7 +7077,7 @@ bool is_double_pointer(Type *t) {
 irValue *ir_emit_source_code_location(irProcedure *proc, String procedure, TokenPos pos) {
 	gbAllocator a = ir_allocator();
 	irValue *v = ir_alloc_value(irValue_SourceCodeLocation);
-	v->SourceCodeLocation.file      = ir_find_or_add_entity_string(proc->module, pos.file);
+	v->SourceCodeLocation.file      = ir_find_or_add_entity_string(proc->module, get_file_path_string(pos.file_id));
 	v->SourceCodeLocation.line      = ir_const_int(pos.line);
 	v->SourceCodeLocation.column    = ir_const_int(pos.column);
 	v->SourceCodeLocation.procedure = ir_find_or_add_entity_string(proc->module, procedure);
@@ -7960,7 +7960,7 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) {
 	TokenPos expr_pos = ast_token(expr).pos;
 
 	TypeAndValue tv = type_and_value_of_expr(expr);
-	GB_ASSERT_MSG(tv.mode != Addressing_Invalid, "invalid expression '%s' @ %.*s(%td:%td)", expr_to_string(expr), LIT(expr_pos.file), expr_pos.line, expr_pos.column);
+	GB_ASSERT_MSG(tv.mode != Addressing_Invalid, "invalid expression '%s' @ %s", expr_to_string(expr), token_pos_to_string(expr_pos));
 	if (tv.mode == Addressing_Type) {
 		// HACK TODO(bill): This is hack but it should be safe in virtually all cases
 		irValue *v = ir_typeid(proc->module, tv.type);
@@ -8018,7 +8018,7 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) {
 				return ir_addr_load(proc, ir_build_addr(proc, expr));
 			}
 
-			GB_PANIC("Error in: %.*s(%td:%td) %s\n", LIT(proc->name), e->token.pos.line, e->token.pos.column);
+			GB_PANIC("Error in: %s %s\n", LIT(proc->name), token_pos_to_string(e->token.pos));
 		}
 
 		return ir_add_module_constant(proc->module, tv.type, tv.value);
@@ -8038,12 +8038,12 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) {
 	switch (expr->kind) {
 	case_ast_node(bl, BasicLit, expr);
 		TokenPos pos = bl->token.pos;
-		GB_PANIC("Non-constant basic literal %.*s(%td:%td) - %.*s", LIT(pos.file), pos.line, pos.column, LIT(token_strings[bl->token.kind]));
+		GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(token_strings[bl->token.kind]));
 	case_end;
 
 	case_ast_node(bd, BasicDirective, expr);
 		TokenPos pos = bd->token.pos;
-		GB_PANIC("Non-constant basic literal %.*s(%td:%td) - %.*s", LIT(pos.file), pos.line, pos.column, LIT(bd->name));
+		GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(bd->name));
 	case_end;
 
 	case_ast_node(i, Implicit, expr);
@@ -8062,8 +8062,8 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) {
 		if (e->kind == Entity_Builtin) {
 			Token token = ast_token(expr);
 			GB_PANIC("TODO(bill): ir_build_expr Entity_Builtin '%.*s'\n"
-			         "\t at %.*s(%td:%td)", LIT(builtin_procs[e->Builtin.id].name),
-			         LIT(token.pos.file), token.pos.line, token.pos.column);
+			         "\t at %s", LIT(builtin_procs[e->Builtin.id].name),
+			         token_pos_to_string(token.pos));
 			return nullptr;
 		} else if (e->kind == Entity_Nil) {
 			return ir_value_nil(tv.type);
@@ -8259,7 +8259,7 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) {
 					auto args = array_make<irValue *>(ir_allocator(), 6);
 					args[0] = ok;
 
-					args[1] = ir_find_or_add_entity_string(proc->module, pos.file);
+					args[1] = ir_find_or_add_entity_string(proc->module, get_file_path_string(pos.file_id));
 					args[2] = ir_const_int(pos.line);
 					args[3] = ir_const_int(pos.column);
 
@@ -8284,7 +8284,7 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) {
 					auto args = array_make<irValue *>(ir_allocator(), 6);
 					args[0] = ok;
 
-					args[1] = ir_find_or_add_entity_string(proc->module, pos.file);
+					args[1] = ir_find_or_add_entity_string(proc->module, get_file_path_string(pos.file_id));
 					args[2] = ir_const_int(pos.line);
 					args[3] = ir_const_int(pos.column);
 
@@ -9649,9 +9649,9 @@ irAddr ir_build_addr(irProcedure *proc, Ast *expr) {
 	TokenPos token_pos = ast_token(expr).pos;
 	GB_PANIC("Unexpected address expression\n"
 	         "\tAst: %.*s @ "
-	         "%.*s(%td:%td)\n",
+	         "%s\n",
 	         LIT(ast_strings[expr->kind]),
-	         LIT(token_pos.file), token_pos.line, token_pos.column);
+	         token_pos_to_string(token_pos));
 
 
 	return ir_addr(nullptr);
@@ -10263,7 +10263,6 @@ irAddr ir_store_range_stmt_val(irProcedure *proc, Ast *stmt_val, irValue *value)
 			// gb_printf_err("%s\n", expr_to_string(stmt_val));
 			// gb_printf_err("Entity: %s -> Value: %s\n", type_to_string(e->type), type_to_string(vt));
 			// Token tok = ast_token(stmt_val);
-			// gb_printf_err("%.*s(%td:%td)\n", LIT(tok.pos.file), tok.pos.line, tok.pos.column);
 		}
 	}
 	ir_addr_store(proc, addr, value);

src/llvm_backend.cpp

@@ -185,7 +185,7 @@ void lb_emit_bounds_check(lbProcedure *p, Token token, lbValue index, lbValue le
 	index = lb_emit_conv(p, index, t_int);
 	len = lb_emit_conv(p, len, t_int);
 
-	lbValue file = lb_find_or_add_entity_string(p->module, token.pos.file);
+	lbValue file = lb_find_or_add_entity_string(p->module, get_file_path_string(token.pos.file_id));
 	lbValue line = lb_const_int(p->module, t_int, token.pos.line);
 	lbValue column = lb_const_int(p->module, t_int, token.pos.column);
 
@@ -207,7 +207,7 @@ void lb_emit_slice_bounds_check(lbProcedure *p, Token token, lbValue low, lbValu
 		return;
 	}
 
-	lbValue file = lb_find_or_add_entity_string(p->module, token.pos.file);
+	lbValue file = lb_find_or_add_entity_string(p->module, get_file_path_string(token.pos.file_id));
 	lbValue line = lb_const_int(p->module, t_int, token.pos.line);
 	lbValue column = lb_const_int(p->module, t_int, token.pos.column);
 	high = lb_emit_conv(p, high, t_int);
@@ -4844,7 +4844,7 @@ lbValue lb_const_value(lbModule *m, Type *type, ExactValue value, bool allow_loc
 			return *found;
 		}
 
-		GB_PANIC("Error in: %.*s(%td:%td), missing procedure %.*s\n", LIT(e->token.pos.file), e->token.pos.line, e->token.pos.column, LIT(e->token.string));
+		GB_PANIC("Error in: %s, missing procedure %.*s\n", token_pos_to_string(e->token.pos), LIT(e->token.string));
 	}
 
 	bool is_local = allow_local && m->curr_procedure != nullptr;
@@ -5414,7 +5414,7 @@ lbValue lb_emit_source_code_location(lbProcedure *p, String const &procedure, To
 	lbModule *m = p->module;
 
 	LLVMValueRef fields[4] = {};
-	fields[0]/*file*/      = lb_find_or_add_entity_string(p->module, pos.file).value;
+	fields[0]/*file*/      = lb_find_or_add_entity_string(p->module, get_file_path_string(pos.file_id)).value;
 	fields[1]/*line*/      = lb_const_int(m, t_int, pos.line).value;
 	fields[2]/*column*/    = lb_const_int(m, t_int, pos.column).value;
 	fields[3]/*procedure*/ = lb_find_or_add_entity_string(p->module, procedure).value;
@@ -9509,7 +9509,7 @@ lbValue lb_emit_union_cast(lbProcedure *p, lbValue value, Type *type, TokenPos p
 			auto args = array_make<lbValue>(permanent_allocator(), 7);
 			args[0] = ok;
 
-			args[1] = lb_const_string(m, pos.file);
+			args[1] = lb_const_string(m, get_file_path_string(pos.file_id));
 			args[2] = lb_const_int(m, t_int, pos.line);
 			args[3] = lb_const_int(m, t_int, pos.column);
 
@@ -9571,7 +9571,7 @@ lbAddr lb_emit_any_cast_addr(lbProcedure *p, lbValue value, Type *type, TokenPos
 		auto args = array_make<lbValue>(permanent_allocator(), 7);
 		args[0] = ok;
 
-		args[1] = lb_const_string(m, pos.file);
+		args[1] = lb_const_string(m, get_file_path_string(pos.file_id));
 		args[2] = lb_const_int(m, t_int, pos.line);
 		args[3] = lb_const_int(m, t_int, pos.column);
 
@@ -9614,7 +9614,7 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) {
 
 	TokenPos expr_pos = ast_token(expr).pos;
 	TypeAndValue tv = type_and_value_of_expr(expr);
-	GB_ASSERT_MSG(tv.mode != Addressing_Invalid, "invalid expression '%s' (tv.mode = %d, tv.type = %s) @ %.*s(%td:%td)\n Current Proc: %.*s : %s", expr_to_string(expr), tv.mode, type_to_string(tv.type), LIT(expr_pos.file), expr_pos.line, expr_pos.column, LIT(p->name), type_to_string(p->type));
+	GB_ASSERT_MSG(tv.mode != Addressing_Invalid, "invalid expression '%s' (tv.mode = %d, tv.type = %s) @ %s\n Current Proc: %.*s : %s", expr_to_string(expr), tv.mode, type_to_string(tv.type), token_pos_to_string(expr_pos), LIT(p->name), type_to_string(p->type));
 
 	if (tv.value.kind != ExactValue_Invalid) {
 		// NOTE(bill): Short on constant values
@@ -9626,12 +9626,12 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) {
 	switch (expr->kind) {
 	case_ast_node(bl, BasicLit, expr);
 		TokenPos pos = bl->token.pos;
-		GB_PANIC("Non-constant basic literal %.*s(%td:%td) - %.*s", LIT(pos.file), pos.line, pos.column, LIT(token_strings[bl->token.kind]));
+		GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(token_strings[bl->token.kind]));
 	case_end;
 
 	case_ast_node(bd, BasicDirective, expr);
 		TokenPos pos = bd->token.pos;
-		GB_PANIC("Non-constant basic literal %.*s(%td:%td) - %.*s", LIT(pos.file), pos.line, pos.column, LIT(bd->name));
+		GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(bd->name));
 	case_end;
 
 	case_ast_node(i, Implicit, expr);
@@ -9658,8 +9658,8 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) {
 		if (e->kind == Entity_Builtin) {
 			Token token = ast_token(expr);
 			GB_PANIC("TODO(bill): lb_build_expr Entity_Builtin '%.*s'\n"
-			         "\t at %.*s(%td:%td)", LIT(builtin_procs[e->Builtin.id].name),
-			         LIT(token.pos.file), token.pos.line, token.pos.column);
+			         "\t at %s", LIT(builtin_procs[e->Builtin.id].name),
+			         token_pos_to_string(token.pos));
 			return {};
 		} else if (e->kind == Entity_Nil) {
 			lbValue res = {};
@@ -9680,7 +9680,7 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) {
 		} else if (e != nullptr && e->kind == Entity_Variable) {
 			return lb_addr_load(p, lb_build_addr(p, expr));
 		}
-		gb_printf_err("Error in: %.*s(%td:%td)\n", LIT(p->name), i->token.pos.line, i->token.pos.column);
+		gb_printf_err("Error in: %s\n", token_pos_to_string(i->token.pos));
 		String pkg = {};
 		if (e->pkg) {
 			pkg = e->pkg->name;
@@ -9877,7 +9877,7 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) {
 					auto args = array_make<lbValue>(permanent_allocator(), 6);
 					args[0] = ok;
 
-					args[1] = lb_find_or_add_entity_string(p->module, pos.file);
+					args[1] = lb_find_or_add_entity_string(p->module, get_file_path_string(pos.file_id));
 					args[2] = lb_const_int(p->module, t_int, pos.line);
 					args[3] = lb_const_int(p->module, t_int, pos.column);
 
@@ -9902,7 +9902,7 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) {
 					auto args = array_make<lbValue>(permanent_allocator(), 6);
 					args[0] = ok;
 
-					args[1] = lb_find_or_add_entity_string(p->module, pos.file);
+					args[1] = lb_find_or_add_entity_string(p->module, get_file_path_string(pos.file_id));
 					args[2] = lb_const_int(p->module, t_int, pos.line);
 					args[3] = lb_const_int(p->module, t_int, pos.column);
 
@@ -11345,9 +11345,9 @@ lbAddr lb_build_addr(lbProcedure *p, Ast *expr) {
 	TokenPos token_pos = ast_token(expr).pos;
 	GB_PANIC("Unexpected address expression\n"
 	         "\tAst: %.*s @ "
-	         "%.*s(%td:%td)\n",
+	         "%s\n",
 	         LIT(ast_strings[expr->kind]),
-	         LIT(token_pos.file), token_pos.line, token_pos.column);
+	         token_pos_to_string(token_pos));
 
 
 	return {};

src/main.cpp

@@ -1845,7 +1845,7 @@ void print_show_unused(Checker *c) {
 		}
 		if (build_context.show_unused_with_location) {
 			TokenPos pos = e->token.pos;
-			print_usage_line(2, "%.*s(%td:%td) %.*s", LIT(pos.file), pos.line, pos.column, LIT(e->token.string));
+			print_usage_line(2, "%s %.*s", token_pos_to_string(pos), LIT(e->token.string));
 		} else {
 			print_usage_line(2, "%.*s", LIT(e->token.string));
 		}

src/parser.cpp

@@ -4455,6 +4455,7 @@ Array<Ast *> parse_stmt_list(AstFile *f) {
 ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
 	GB_ASSERT(f != nullptr);
 	f->fullpath = string_trim_whitespace(fullpath); // Just in case
+	set_file_path_string(f->id, fullpath);
 	if (!string_ends_with(f->fullpath, str_lit(".odin"))) {
 		return ParseFile_WrongExtension;
 	}
@@ -4462,6 +4463,10 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
 	if (build_context.insert_semicolon) {
 		tokenizer_flags = TokenizerFlag_InsertSemicolon;
 	}
+
+	zero_item(&f->tokenizer);
+	f->tokenizer.curr_file_id = f->id;
+
 	TokenizerInitError err = init_tokenizer(&f->tokenizer, f->fullpath, tokenizer_flags);
 	if (err != TokenizerInit_None) {
 		switch (err) {
@@ -4471,6 +4476,8 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
 			return ParseFile_NotFound;
 		case TokenizerInit_Permission:
 			return ParseFile_Permission;
+		case TokenizerInit_FileTooLarge:
+			return ParseFile_FileTooLarge;
 		default:
 			return ParseFile_InvalidFile;
 		}
@@ -4490,9 +4497,9 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
 
 	if (err == TokenizerInit_Empty) {
 		Token token = {Token_EOF};
-		token.pos.file   = fullpath;
-		token.pos.line   = 1;
-		token.pos.column = 1;
+		token.pos.file_id = f->id;
+		token.pos.line    = 1;
+		token.pos.column  = 1;
 		array_add(&f->tokens, token);
 		return ParseFile_None;
 	}
@@ -4593,7 +4600,7 @@ void parser_add_package(Parser *p, AstPackage *pkg) {
 			syntax_error(f->package_token, "Non-unique package name '%.*s'", LIT(pkg->name));
 			GB_ASSERT((*found)->files.count > 0);
 			TokenPos pos = (*found)->files[0]->package_token.pos;
-			error_line("\tpreviously declared at %.*s(%td:%td)\n", LIT(pos.file), pos.line, pos.column);
+			error_line("\tpreviously declared at %s\n", token_pos_to_string(pos));
 		} else {
 			string_map_set(&p->package_map, key, pkg);
 		}
@@ -5227,11 +5234,11 @@ ParseFileError process_imported_file(Parser *p, ImportedFile const &imported_fil
 
 	AstFile *file = gb_alloc_item(heap_allocator(), AstFile);
 	file->pkg = pkg;
-	file->id = imported_file.index+1;
+	file->id = cast(i32)(imported_file.index+1);
 
 	TokenPos err_pos = {0};
 	ParseFileError err = init_ast_file(file, fi->fullpath, &err_pos);
-	err_pos.file = fi->fullpath;
+	err_pos.file_id = file->id;
 	file->last_error = err;
 
 	if (err != ParseFile_None) {
@@ -5260,6 +5267,9 @@ ParseFileError process_imported_file(Parser *p, ImportedFile const &imported_fil
 			case ParseFile_EmptyFile:
 				syntax_error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi->name));
 				break;
+			case ParseFile_FileTooLarge:
+				syntax_error(pos, "Failed to parse file: %.*s; file is too large, exceeds maximum file size of 2 GiB", LIT(fi->name));
+				break;
 			}
 
 			return err;

src/parser.hpp

@@ -42,6 +42,7 @@ enum ParseFileError {
 	ParseFile_NotFound,
 	ParseFile_InvalidToken,
 	ParseFile_GeneralError,
+	ParseFile_FileTooLarge,
 
 	ParseFile_Count,
 };
@@ -74,7 +75,7 @@ struct ImportedFile {
 };
 
 struct AstFile {
-	isize        id;
+	i32          id;
 	AstPackage * pkg;
 	Scope *      scope;
 

src/query_data.cpp

@@ -553,7 +553,7 @@ void generate_and_print_query_data_global_definitions(Checker *c, Timings *timin
 
 			def->add("package",     e->pkg->name);
 			def->add("name",        name);
-			def->add("filepath",    e->token.pos.file);
+			def->add("filepath",    get_file_path_string(e->token.pos.file_id));
 			def->add("line",        cast(i64)e->token.pos.line);
 			def->add("column",      cast(i64)e->token.pos.column);
 			def->add("file_offset", cast(i64)e->token.pos.offset);
@@ -915,7 +915,7 @@ void generate_and_print_query_data_go_to_definitions(Checker *c) {
 		}
 
 
-		AstFile **use_file_found = string_map_get(&c->info.files, pos.file);
+		AstFile **use_file_found = string_map_get(&c->info.files, get_file_path_string(pos.file_id));
 		GB_ASSERT(use_file_found != nullptr);
 		AstFile *use_file = *use_file_found;
 		GB_ASSERT(use_file != nullptr);
@@ -1005,7 +1005,7 @@ void generate_and_print_query_data_go_to_definitions(Checker *c) {
 			AstFile *def_file = e->file;
 
 			if (def_file == nullptr) {
-				auto *def_file_found = string_map_get(&c->info.files, e->token.pos.file);
+				auto *def_file_found = string_map_get(&c->info.files, get_file_path_string(e->token.pos.file_id));
 				if (def_file_found == nullptr) {
 					continue;
 				}

src/tokenizer.cpp

@@ -185,14 +185,25 @@ void init_keyword_hash_table(void) {
 	GB_ASSERT(max_keyword_size < 16);
 }
 
+gb_global Array<String> global_file_path_strings; // index is file id
+
+String get_file_path_string(i32 index);
 
 struct TokenPos {
-	String file;
-	isize  offset; // starting at 0
-	isize  line;   // starting at 1
-	isize  column; // starting at 1
+	i32 file_id;
+	i32 offset; // starting at 0
+	i32 line;   // starting at 1
+	i32 column; // starting at 1
 };
 
+// temporary
+char *token_pos_to_string(TokenPos const &pos) {
+	gbString s = gb_string_make_reserve(temporary_allocator(), 128);
+	String file = get_file_path_string(pos.file_id);
+	s = gb_string_append_fmt(s, "%.*s(%d:%d)", LIT(file), pos.line, pos.column);
+	return s;
+}
+
 i32 token_pos_cmp(TokenPos const &a, TokenPos const &b) {
 	if (a.offset != b.offset) {
 		return (a.offset < b.offset) ? -1 : +1;
@@ -203,7 +214,7 @@ i32 token_pos_cmp(TokenPos const &a, TokenPos const &b) {
 	if (a.column != b.column) {
 		return (a.column < b.column) ? -1 : +1;
 	}
-	return string_compare(a.file, b.file);
+	return string_compare(get_file_path_string(a.file_id), get_file_path_string(b.file_id));
 }
 
 bool operator==(TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a, b) == 0; }
@@ -238,6 +249,7 @@ struct ErrorCollector {
 	i64     warning_count;
 	bool    in_block;
 	gbMutex mutex;
+	gbMutex string_mutex;
 
 	Array<u8> error_buffer;
 	Array<String> errors;
@@ -254,11 +266,44 @@ bool any_errors(void) {
 
 void init_global_error_collector(void) {
 	gb_mutex_init(&global_error_collector.mutex);
+	gb_mutex_init(&global_error_collector.string_mutex);
 	array_init(&global_error_collector.errors, heap_allocator());
 	array_init(&global_error_collector.error_buffer, heap_allocator());
+	array_init(&global_file_path_strings, heap_allocator(), 4096);
 }
 
 
+bool set_file_path_string(i32 index, String const &path) {
+	bool ok = false;
+	GB_ASSERT(index >= 0);
+	gb_mutex_lock(&global_error_collector.string_mutex);
+
+	if (index >= global_file_path_strings.count) {
+		array_resize(&global_file_path_strings, index);
+	}
+	String prev = global_file_path_strings[index];
+	if (prev.len == 0) {
+		global_file_path_strings[index] = path;
+		ok = true;
+	}
+
+	gb_mutex_unlock(&global_error_collector.string_mutex);
+	return ok;
+}
+
+String get_file_path_string(i32 index) {
+	GB_ASSERT(index >= 0);
+	gb_mutex_lock(&global_error_collector.string_mutex);
+
+	String path = {};
+	if (index < global_file_path_strings.count) {
+		path = global_file_path_strings[index];
+	}
+
+	gb_mutex_unlock(&global_error_collector.string_mutex);
+	return path;
+}
+
 void begin_error_block(void) {
 	gb_mutex_lock(&global_error_collector.mutex);
 	global_error_collector.in_block = true;
@@ -335,8 +380,8 @@ void error_va(Token token, char const *fmt, va_list va) {
 		error_out("Error: %s\n", gb_bprintf_va(fmt, va));
 	} else if (global_error_collector.prev != token.pos) {
 		global_error_collector.prev = token.pos;
-		error_out("%.*s(%td:%td) %s\n",
-		          LIT(token.pos.file), token.pos.line, token.pos.column,
+		error_out("%s %s\n",
+		          token_pos_to_string(token.pos),
 		          gb_bprintf_va(fmt, va));
 	}
 	gb_mutex_unlock(&global_error_collector.mutex);
@@ -358,8 +403,8 @@ void warning_va(Token token, char const *fmt, va_list va) {
 			error_out("Warning: %s\n", gb_bprintf_va(fmt, va));
 		} else if (global_error_collector.prev != token.pos) {
 			global_error_collector.prev = token.pos;
-			error_out("%.*s(%td:%td) Warning: %s\n",
-			          LIT(token.pos.file), token.pos.line, token.pos.column,
+			error_out("%s Warning: %s\n",
+			          token_pos_to_string(token.pos),
 			          gb_bprintf_va(fmt, va));
 		}
 	}
@@ -381,8 +426,8 @@ void error_no_newline_va(Token token, char const *fmt, va_list va) {
 		error_out("Error: %s", gb_bprintf_va(fmt, va));
 	} else if (global_error_collector.prev != token.pos) {
 		global_error_collector.prev = token.pos;
-		error_out("%.*s(%td:%td) %s",
-		          LIT(token.pos.file), token.pos.line, token.pos.column,
+		error_out("%s %s",
+		          token_pos_to_string(token.pos),
 		          gb_bprintf_va(fmt, va));
 	}
 	gb_mutex_unlock(&global_error_collector.mutex);
@@ -398,9 +443,9 @@ void syntax_error_va(Token token, char const *fmt, va_list va) {
 	// NOTE(bill): Duplicate error, skip it
 	if (global_error_collector.prev != token.pos) {
 		global_error_collector.prev = token.pos;
-		error_out("%.*s(%td:%td) Syntax Error: %s\n",
-		              LIT(token.pos.file), token.pos.line, token.pos.column,
-		              gb_bprintf_va(fmt, va));
+		error_out("%s Syntax Error: %s\n",
+		          token_pos_to_string(token.pos),
+		          gb_bprintf_va(fmt, va));
 	} else if (token.pos.line == 0) {
 		error_out("Syntax Error: %s\n", gb_bprintf_va(fmt, va));
 	}
@@ -422,8 +467,8 @@ void syntax_warning_va(Token token, char const *fmt, va_list va) {
 		// NOTE(bill): Duplicate error, skip it
 		if (global_error_collector.prev != token.pos) {
 			global_error_collector.prev = token.pos;
-			error_out("%.*s(%td:%td) Syntax Warning: %s\n",
-			          LIT(token.pos.file), token.pos.line, token.pos.column,
+			error_out("%S Syntax Warning: %s\n",
+			          token_pos_to_string(token.pos),
 			          gb_bprintf_va(fmt, va));
 		} else if (token.pos.line == 0) {
 			error_out("Warning: %s\n", gb_bprintf_va(fmt, va));
@@ -529,6 +574,7 @@ enum TokenizerInitError {
 	TokenizerInit_NotExists,
 	TokenizerInit_Permission,
 	TokenizerInit_Empty,
+	TokenizerInit_FileTooLarge,
 
 	TokenizerInit_Count,
 };
@@ -539,7 +585,7 @@ struct TokenizerState {
 	u8 *  curr;        // character pos
 	u8 *  read_curr;   // pos from start
 	u8 *  line;        // current line pos
-	isize line_count;
+	i32   line_count;
 	bool  insert_semicolon;
 };
 
@@ -549,6 +595,7 @@ enum TokenizerFlags {
 };
 
 struct Tokenizer {
+	i32 curr_file_id;
 	String fullpath;
 	u8 *start;
 	u8 *end;
@@ -557,9 +604,9 @@ struct Tokenizer {
 	u8 *  curr;        // character pos
 	u8 *  read_curr;   // pos from start
 	u8 *  line;        // current line pos
-	isize line_count;
+	i32   line_count;
 
-	isize error_count;
+	i32 error_count;
 	Array<String> allocated_strings;
 
 	TokenizerFlags flags;
@@ -595,9 +642,9 @@ void tokenizer_err(Tokenizer *t, char const *msg, ...) {
 		column = 1;
 	}
 	Token token = {};
-	token.pos.file = t->fullpath;
+	token.pos.file_id = t->curr_file_id;
 	token.pos.line = t->line_count;
-	token.pos.column = column;
+	token.pos.column = cast(i32)column;
 
 	va_start(va, msg);
 	syntax_error_va(token, msg, va);
@@ -647,13 +694,15 @@ TokenizerInitError init_tokenizer(Tokenizer *t, String fullpath, TokenizerFlags
 
 	// TODO(bill): Memory map rather than copy contents
 	gbFileContents fc = gb_file_read_contents(heap_allocator(), true, c_str);
-	gb_zero_item(t);
 
 	t->flags = flags;
 	t->fullpath = fullpath;
 	t->line_count = 1;
 
-	if (fc.data != nullptr) {
+	if (fc.size > I32_MAX) {
+		err = TokenizerInit_FileTooLarge;
+		gb_file_free_contents(&fc);
+	} else if (fc.data != nullptr) {
 		t->start = cast(u8 *)fc.data;
 		t->line = t->read_curr = t->curr = t->start;
 		t->end = t->start + fc.size;
@@ -721,9 +770,9 @@ u8 peek_byte(Tokenizer *t, isize offset=0) {
 void scan_number_to_token(Tokenizer *t, Token *token, bool seen_decimal_point) {
 	token->kind = Token_Integer;
 	token->string = {t->curr, 1};
-	token->pos.file = t->fullpath;
+	token->pos.file_id = t->curr_file_id;
 	token->pos.line = t->line_count;
-	token->pos.column = t->curr-t->line+1;
+	token->pos.column = cast(i32)(t->curr-t->line+1);
 
 	if (seen_decimal_point) {
 		token->string.text -= 1;
@@ -930,11 +979,10 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
 	token->kind = Token_Invalid;
 	token->string.text = t->curr;
 	token->string.len  = 1;
-	token->pos.file.text = t->fullpath.text;
-	token->pos.file.len  = t->fullpath.len;
+	token->pos.file_id = t->curr_file_id;
 	token->pos.line = t->line_count;
-	token->pos.offset = t->curr - t->start;
-	token->pos.column = t->curr - t->line + 1;
+	token->pos.offset = cast(i32)(t->curr - t->start);
+	token->pos.column = cast(i32)(t->curr - t->line + 1);
 
 	bool insert_semicolon = false;