Experimental try for ABI for return values on windows

It's all done by reverse engineering it. I may be wrong...

src/check_expr.c

@@ -1217,6 +1217,29 @@ Type *type_to_abi_compat_result_type(gbAllocator a, Type *original_type) {
 	return new_type;
 }
 
+bool abi_compat_return_by_value(gbAllocator a, ProcCallingConvention cc, Type *abi_return_type) {
+	if (abi_return_type == NULL) {
+		return false;
+	}
+	if (cc == ProcCC_Odin) {
+		return false;
+	}
+
+	if (str_eq(build_context.ODIN_OS, str_lit("windows"))) {
+		i64 size = 8*type_size_of(a, abi_return_type);
+		switch (size) {
+		case 0:
+		case 8:
+		case 16:
+		case 32:
+		case 64:
+			return false;
+		default:
+			return true;
+		}
+	}
+	return false;
+}
 
 void check_procedure_type(Checker *c, Type *type, AstNode *proc_type_node) {
 	ast_node(pt, ProcType, proc_type_node);
@@ -1248,6 +1271,7 @@ void check_procedure_type(Checker *c, Type *type, AstNode *proc_type_node) {
 
 	// NOTE(bill): The types are the same
 	type->Proc.abi_compat_result_type = type_to_abi_compat_result_type(c->allocator, type->Proc.results);
+	type->Proc.return_by_pointer = abi_compat_return_by_value(c->allocator, pt->calling_convention, type->Proc.abi_compat_result_type);
 }
 
 

src/entity.c

@@ -43,6 +43,7 @@ typedef enum EntityFlag {
 	EntityFlag_NoAlias    = 1<<7,
 	EntityFlag_TypeField  = 1<<8,
 	EntityFlag_Value      = 1<<9,
+	EntityFlag_Sret       = 1<<10,
 } EntityFlag;
 
 // Zero value means the overloading process is not yet done

src/ir.c

@@ -121,6 +121,7 @@ struct irProcedure {
 	AstNode *             body;
 	u64                   tags;
 
+	irValue *             return_ptr;
 	irValueArray          params;
 	Array(irDefer)        defer_stmts;
 	Array(irBlock *)      blocks;
@@ -219,6 +220,7 @@ struct irProcedure {
 	IR_INSTR_KIND(Call, struct {                                      \
 		Type *    type; /* return type */                             \
 		irValue * value;                                              \
+		irValue * return_ptr;                                         \
 		irValue **args;                                               \
 		isize     arg_count;                                          \
 	})                                                                \
@@ -984,9 +986,10 @@ irValue *ir_instr_select(irProcedure *p, irValue *cond, irValue *t, irValue *f)
 	return v;
 }
 
-irValue *ir_instr_call(irProcedure *p, irValue *value, irValue **args, isize arg_count, Type *result_type) {
+irValue *ir_instr_call(irProcedure *p, irValue *value, irValue *return_ptr, irValue **args, isize arg_count, Type *result_type) {
 	irValue *v = ir_alloc_instr(p, irInstr_Call);
 	v->Instr.Call.value = value;
+	v->Instr.Call.return_ptr = return_ptr;
 	v->Instr.Call.args = args;
 	v->Instr.Call.arg_count = arg_count;
 	v->Instr.Call.type = result_type;
@@ -1491,8 +1494,14 @@ irValue *ir_emit_call(irProcedure *p, irValue *value, irValue **args, isize arg_
 
 	Type *abi_rt = pt->Proc.abi_compat_result_type;
 	Type *rt = reduce_tuple_to_single_type(results);
+	if (pt->Proc.return_by_pointer) {
+		irValue *return_ptr = ir_add_local_generated(p, rt);
+		GB_ASSERT(is_type_pointer(ir_type(return_ptr)));
+		ir_emit(p, ir_instr_call(p, value, return_ptr, args, arg_count, NULL));
+		return ir_emit_load(p, return_ptr);
+	}
 
-	irValue *result = ir_emit(p, ir_instr_call(p, value, args, arg_count, abi_rt));
+	irValue *result = ir_emit(p, ir_instr_call(p, value, NULL, args, arg_count, abi_rt));
 	if (abi_rt != results) {
 		result = ir_emit_transmute(p, result, rt);
 	}
@@ -1555,12 +1564,17 @@ void ir_emit_unreachable(irProcedure *proc) {
 void ir_emit_return(irProcedure *proc, irValue *v) {
 	ir_emit_defer_stmts(proc, irDeferExit_Return, NULL);
 
-	Type *abi_rt = proc->type->Proc.abi_compat_result_type;
-	if (abi_rt != proc->type->Proc.results) {
-		v = ir_emit_transmute(proc, v, abi_rt);
-	}
+	if (proc->type->Proc.return_by_pointer) {
+		ir_emit_store(proc, proc->return_ptr, v);
+		ir_emit(proc, ir_instr_return(proc, NULL));
+	} else {
+		Type *abi_rt = proc->type->Proc.abi_compat_result_type;
+		if (abi_rt != proc->type->Proc.results) {
+			v = ir_emit_transmute(proc, v, abi_rt);
+		}
 
-	ir_emit(proc, ir_instr_return(proc, v));
+		ir_emit(proc, ir_instr_return(proc, v));
+	}
 }
 
 void ir_emit_jump(irProcedure *proc, irBlock *target_block) {
@@ -6718,6 +6732,20 @@ void ir_begin_procedure_body(irProcedure *proc) {
 	proc->entry_block = ir_new_block(proc, proc->type_expr, "entry");
 	ir_start_block(proc, proc->entry_block);
 
+	if (proc->type->Proc.return_by_pointer) {
+		// NOTE(bill): this must be the first parameter stored
+		gbAllocator a = proc->module->allocator;
+		Type *ptr_type = make_type_pointer(a, reduce_tuple_to_single_type(proc->type->Proc.results));
+		Entity *e = make_entity_param(a, NULL, make_token_ident(str_lit("agg.result")), ptr_type, false, false);
+		e->flags |= EntityFlag_Sret | EntityFlag_NoAlias;
+
+		irValue *param = ir_value_param(a, proc, e, ptr_type);
+		param->Param.kind = irParamPass_Pointer;
+
+		ir_module_add_value(proc->module, e, param);
+		proc->return_ptr = param;
+	}
+
 	if (proc->type->Proc.params != NULL) {
 		ast_node(pt, ProcType, proc->type_expr);
 		isize param_index = 0;
@@ -6744,6 +6772,8 @@ void ir_begin_procedure_body(irProcedure *proc) {
 			}
 		}
 	}
+
+
 }
 
 

src/ir_print.c

@@ -142,7 +142,7 @@ void ir_print_proc_results(irFileBuffer *f, irModule *m, Type *t) {
 	GB_ASSERT(is_type_proc(t));
 	t = base_type(t);
 	isize result_count = t->Proc.result_count;
-	if (result_count == 0) {
+	if (result_count == 0 || t->Proc.return_by_pointer) {
 		ir_fprintf(f, "void");
 	} else {
 		Type *rt = t->Proc.abi_compat_result_type;
@@ -322,6 +322,13 @@ void ir_print_type(irFileBuffer *f, irModule *m, Type *t) {
 		isize result_count = t->Proc.result_count;
 		ir_print_proc_results(f, m, t);
 		ir_fprintf(f, " (");
+		if (t->Proc.return_by_pointer) {
+			ir_print_type(f, m, reduce_tuple_to_single_type(t->Proc.results));
+			ir_fprintf(f, "* sret noalias ");
+			if (param_count > 0) {
+				ir_fprintf(f, ", ");
+			}
+		}
 		for (isize i = 0; i < param_count; i++) {
 			if (i > 0) {
 				ir_fprintf(f, ", ");
@@ -1248,7 +1255,7 @@ void ir_print_instr(irFileBuffer *f, irModule *m, irValue *value) {
 		}
 		ir_fprintf(f, "call ");
 		ir_print_calling_convention(f, m, proc_type->Proc.calling_convention);
-		if (result_type) {
+		if (result_type && !proc_type->Proc.return_by_pointer) {
 			ir_print_proc_results(f, m, proc_type);
 		} else {
 			ir_fprintf(f, "void");
@@ -1258,6 +1265,16 @@ void ir_print_instr(irFileBuffer *f, irModule *m, irValue *value) {
 
 
 		ir_fprintf(f, "(");
+		if (proc_type->Proc.return_by_pointer) {
+			GB_ASSERT(call->return_ptr != NULL);
+			ir_print_type(f, m, proc_type->Proc.results);
+			ir_fprintf(f, "* ");
+			ir_print_value(f, m, call->return_ptr, ir_type(call->return_ptr));
+			if (call->arg_count > 0) {
+				ir_fprintf(f, ", ");
+			}
+		}
+
 		if (call->arg_count > 0) {
 			Type *proc_type = base_type(ir_type(call->value));
 			GB_ASSERT(proc_type->kind == Type_Proc);
@@ -1493,6 +1510,15 @@ void ir_print_proc(irFileBuffer *f, irModule *m, irProcedure *proc) {
 
 	ir_fprintf(f, "(");
 
+	if (proc_type->return_by_pointer) {
+		ir_print_type(f, m, reduce_tuple_to_single_type(proc_type->results));
+		ir_fprintf(f, "* sret noalias ");
+		ir_fprintf(f, "%%agg.result");
+		if (param_count > 0) {
+			ir_fprintf(f, ", ");
+		}
+	}
+
 	if (param_count > 0) {
 		TypeTuple *params = &proc_type->params->Tuple;
 		for (isize i = 0; i < params->variable_count; i++) {

src/types.c

@@ -137,6 +137,7 @@ typedef struct TypeRecord {
 		Type * results; /* Type_Tuple */                  \
 		i32    param_count;                               \
 		i32    result_count;                              \
+		bool   return_by_pointer;                         \
 		Type **abi_compat_params;                         \
 		Type * abi_compat_result_type;                    \
 		bool   variadic;                                  \