Merge pull request #5347 from Feoramund/test-expect-assert

Let tests expect assertion failures and signals raised

core/testing/runner.odin

@@ -741,7 +741,8 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 
 
 		if test_index, reason, ok := should_stop_test(); ok {
-			#no_bounds_check report.all_test_states[test_index] = .Failed
+			passed := reason == .Successful_Stop
+			#no_bounds_check report.all_test_states[test_index] = .Successful if passed else .Failed
 			#no_bounds_check it := internal_tests[test_index]
 			#no_bounds_check pkg := report.packages_by_name[it.pkg]
 			pkg.frame_ready = false
@@ -762,7 +763,7 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 			fmt.assertf(task_data != nil, "A signal (%v) was raised to stop test #%i %s.%s, but its task data is missing.",
 				reason, test_index, it.pkg, it.name)
 
-			if !task_data.t._fail_now_called {
+			if !passed && !task_data.t._fail_now_called {
 				if test_index not_in failed_test_reason_map {
 					// We only write a new error message here if there wasn't one
 					// already, because the message we can provide based only on
@@ -780,7 +781,11 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 
 			end_t(&task_data.t)
 
-			total_failure_count += 1
+			if passed {
+				total_success_count += 1
+			} else {
+				total_failure_count += 1
+			}
 			total_done_count += 1
 		}
 

core/testing/signal_handler.odin

@@ -12,8 +12,26 @@ package testing
 import "base:runtime"
 import "core:log"
 
+@(private, thread_local)
+local_test_expected_failures: struct {
+	signal:         i32,
+
+	message_count:  int,
+	messages:       [MAX_EXPECTED_ASSERTIONS_PER_TEST]string,
+
+	location_count: int,
+	locations:      [MAX_EXPECTED_ASSERTIONS_PER_TEST]runtime.Source_Code_Location,
+}
+
+@(private, thread_local)
+local_test_assertion_raised: struct {
+	message: string,
+	location: runtime.Source_Code_Location,
+}
+
 Stop_Reason :: enum {
 	Unknown,
+	Successful_Stop,
 	Illegal_Instruction,
 	Arithmetic_Error,
 	Segmentation_Fault,
@@ -21,7 +39,12 @@ Stop_Reason :: enum {
 }
 
 test_assertion_failure_proc :: proc(prefix, message: string, loc: runtime.Source_Code_Location) -> ! {
-	log.fatalf("%s: %s", prefix, message, location = loc)
+	if local_test_expected_failures.message_count + local_test_expected_failures.location_count > 0 {
+		local_test_assertion_raised = { message, loc }
+		log.debugf("%s\n\tmessage: %q\n\tlocation: %w", prefix, message, loc)
+	} else {
+		log.fatalf("%s: %s", prefix, message, location = loc)
+	}
 	runtime.trap()
 }
 

core/testing/signal_handler_libc.odin

@@ -20,7 +20,8 @@ import "core:terminal/ansi"
 
 @(private="file") stop_test_gate:   sync.Mutex
 @(private="file") stop_test_index:  libc.sig_atomic_t
-@(private="file") stop_test_reason: libc.sig_atomic_t
+@(private="file") stop_test_signal: libc.sig_atomic_t
+@(private="file") stop_test_passed: libc.sig_atomic_t
 @(private="file") stop_test_alert:  libc.sig_atomic_t
 
 @(private="file", thread_local)
@@ -99,7 +100,30 @@ This is a dire bug and should be reported to the Odin developers.
 
 	if sync.mutex_guard(&stop_test_gate) {
 		intrinsics.atomic_store(&stop_test_index, local_test_index)
-		intrinsics.atomic_store(&stop_test_reason, cast(libc.sig_atomic_t)sig)
+		intrinsics.atomic_store(&stop_test_signal, cast(libc.sig_atomic_t)sig)
+		passed: bool
+		check_passing: {
+			if location := local_test_assertion_raised.location; location != {} {
+				for i in 0..<local_test_expected_failures.location_count {
+					if local_test_expected_failures.locations[i] == location {
+						passed = true
+						break check_passing
+					}
+				}
+			}
+			if message := local_test_assertion_raised.message; message != "" {
+				for i in 0..<local_test_expected_failures.message_count {
+					if local_test_expected_failures.messages[i] == message {
+						passed = true
+						break check_passing
+					}
+				}
+			}
+			if signal := local_test_expected_failures.signal; signal == sig {
+				passed = true
+			}
+		}
+		intrinsics.atomic_store(&stop_test_passed, cast(libc.sig_atomic_t)passed)
 		intrinsics.atomic_store(&stop_test_alert, 1)
 
 		for {
@@ -154,11 +178,15 @@ _should_stop_test :: proc() -> (test_index: int, reason: Stop_Reason, ok: bool)
 		intrinsics.atomic_store(&stop_test_alert, 0)
 
 		test_index = cast(int)intrinsics.atomic_load(&stop_test_index)
-		switch intrinsics.atomic_load(&stop_test_reason) {
-		case libc.SIGFPE: reason = .Arithmetic_Error
-		case libc.SIGILL: reason = .Illegal_Instruction
-		case libc.SIGSEGV: reason = .Segmentation_Fault
-		case      SIGTRAP: reason = .Unhandled_Trap
+		if cast(bool)intrinsics.atomic_load(&stop_test_passed) {
+			reason = .Successful_Stop
+		} else {
+			switch intrinsics.atomic_load(&stop_test_signal) {
+			case libc.SIGFPE: reason = .Arithmetic_Error
+			case libc.SIGILL: reason = .Illegal_Instruction
+			case libc.SIGSEGV: reason = .Segmentation_Fault
+			case      SIGTRAP: reason = .Unhandled_Trap
+			}
 		}
 		ok = true
 	}

core/testing/testing.odin

@@ -21,6 +21,8 @@ import "core:mem"
 _ :: reflect // alias reflect to nothing to force visibility for -vet
 _ :: mem     // in case TRACKING_MEMORY is not enabled
 
+MAX_EXPECTED_ASSERTIONS_PER_TEST :: 5
+
 // IMPORTANT NOTE: Compiler requires this layout
 Test_Signature :: proc(^T)
 
@@ -155,3 +157,74 @@ set_fail_timeout :: proc(t: ^T, duration: time.Duration, loc := #caller_location
 		location = loc,
 	})
 }
+
+/*
+Let the test runner know that it should expect an assertion failure from a
+specific location in the source code for this test.
+
+In the event that an assertion fails, a debug message will be logged with its
+exact message and location in a copyable format to make it convenient to write
+tests which use this API.
+
+This procedure may be called up to 5 times with different locations.
+
+This is a limitation for the sake of simplicity in the implementation, and you
+should consider breaking up your tests into smaller procedures if you need to
+check for asserts in more than 2 places.
+*/
+expect_assert_from :: proc(t: ^T, expected_place: runtime.Source_Code_Location, caller_loc := #caller_location) {
+	count := local_test_expected_failures.location_count
+	if count == MAX_EXPECTED_ASSERTIONS_PER_TEST {
+		panic("This test cannot handle that many expected assertions based on matching the location.", caller_loc)
+	}
+	local_test_expected_failures.locations[count] = expected_place
+	local_test_expected_failures.location_count += 1
+}
+
+/*
+Let the test runner know that it should expect an assertion failure with a
+specific message for this test.
+
+In the event that an assertion fails, a debug message will be logged with its
+exact message and location in a copyable format to make it convenient to write
+tests which use this API.
+
+This procedure may be called up to 5 times with different messages.
+
+This is a limitation for the sake of simplicity in the implementation, and you
+should consider breaking up your tests into smaller procedures if you need to
+check for more than a couple different assertion messages.
+*/
+expect_assert_message :: proc(t: ^T, expected_message: string, caller_loc := #caller_location) {
+	count := local_test_expected_failures.message_count
+	if count == MAX_EXPECTED_ASSERTIONS_PER_TEST {
+		panic("This test cannot handle that many expected assertions based on matching the message.", caller_loc)
+	}
+	local_test_expected_failures.messages[count] = expected_message
+	local_test_expected_failures.message_count += 1
+}
+
+expect_assert :: proc {
+	expect_assert_from,
+	expect_assert_message,
+}
+
+/*
+Let the test runner know that it should expect a signal to be raised within
+this test.
+
+This API is for advanced users, as arbitrary signals will not be caught; only
+the ones already handled by the test runner, such as
+
+- SIGINT,                           (interrupt)
+- SIGTERM,                          (polite termination)
+- SIGILL,                           (illegal instruction)
+- SIGFPE,                           (arithmetic error)
+- SIGSEGV, and                      (segmentation fault)
+- SIGTRAP (only on POSIX systems).  (trap / debug trap)
+
+Note that only one signal can be expected per test.
+*/
+expect_signal :: proc(t: ^T, #any_int sig: i32) {
+	local_test_expected_failures.signal = sig
+}

tests/core/testing/test_core_testing.odin

@@ -0,0 +1,52 @@
+package test_core_testing
+
+import "core:c/libc"
+import "core:math/rand"
+import "core:testing"
+
+@test
+test_expected_assert :: proc(t: ^testing.T) {
+	target := #location(); target.line += 2; target.column = 2
+	testing.expect_assert_from(t, target)
+	assert(false)
+}
+
+@test
+test_expected_two_assert :: proc(t: ^testing.T) {
+	target1 := #location(); target1.line += 5; target1.column = 3
+	target2 := #location(); target2.line += 6; target2.column = 3
+	testing.expect_assert_from(t, target1)
+	testing.expect_assert_from(t, target2)
+	if rand.uint32() & 1 == 0 {
+		assert(false)
+	} else {
+		assert(false)
+	}
+}
+
+some_proc :: proc() {
+	assert(false)
+}
+
+@test
+test_expected_assert_in_proc :: proc(t: ^testing.T) {
+	target := #location(some_proc)
+	target.line += 1
+	target.column = 2
+	assert(target.procedure == "", "The bug's been fixed; this line and the next can be deleted.")
+	target.procedure = "some_proc" // TODO: Is this supposed to be blank on #location(...)?
+	testing.expect_assert(t, target)
+	some_proc()
+}
+
+@test
+test_expected_assert_message :: proc(t: ^testing.T) {
+	testing.expect_assert(t, "failure")
+	assert(false, "failure")
+}
+
+@test
+test_expected_signal :: proc(t: ^testing.T) {
+	testing.expect_signal(t, libc.SIGILL)
+	libc.raise(libc.SIGILL)
+}