o3de/Tools/LyTestTools/tests/unit/test_o3de_multi_test_framework.py

"""
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.

SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import unittest

import pytest
import unittest.mock as mock

import ly_test_tools
import ly_test_tools.launchers.exceptions
import ly_test_tools.o3de.multi_test_framework as multi_test_framework
from ly_test_tools.o3de.editor_test_utils import prepare_asset_processor
from ly_test_tools.launchers.platforms.linux.launcher import LinuxAtomToolsLauncher, LinuxEditor
from ly_test_tools.launchers.platforms.win.launcher import WinAtomToolsLauncher, WinEditor


pytestmark = pytest.mark.SUITE_smoke


class TestResultType(unittest.TestCase):

    class DummySubclass(multi_test_framework.Result.ResultType):
        def __str__(self):
            return None

    def setUp(self):
        self.mock_result = TestResultType.DummySubclass()

    def test_GetOutputStr_HasOutput_ReturnsCorrectly(self):
        self.mock_result.output = 'expected output'
        assert self.mock_result.get_output_str() == 'expected output'

    def test_GetOutputStr_NoOutput_ReturnsCorrectly(self):
        self.mock_result.output = None
        assert self.mock_result.get_output_str() == '-- No output --'

    def test_GetLogStr_HasOutput_ReturnsCorrectly(self):
        self.mock_result.log_output = 'expected log output'
        assert self.mock_result.get_log_output_str() == 'expected log output'

    def test_GetLogStr_NoOutput_ReturnsCorrectly(self):
        self.mock_result.log_output = None
        assert self.mock_result.get_log_output_str() == '-- No log found --'


class TestResultPass(unittest.TestCase):

    def test_Create_ValidArgs_CorrectAttributes(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'

        under_test = multi_test_framework.Result.Pass(mock_test_spec, mock_output, mock_log_output)
        assert mock_test_spec == under_test.test_spec
        assert mock_output == under_test.output
        assert mock_log_output == under_test.log_output

    def test_Str_ValidPassString_ReturnsOutput(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'

        under_test = multi_test_framework.Result.Pass(mock_test_spec, mock_output, mock_log_output)
        assert mock_output in str(under_test)


class TestResultFail(unittest.TestCase):

    def test_Create_ValidArgs_CorrectAttributes(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'

        under_test = multi_test_framework.Result.Fail(mock_test_spec, mock_output, mock_log_output)
        assert mock_test_spec == under_test.test_spec
        assert mock_output == under_test.output
        assert mock_log_output == under_test.log_output

    def test_Str_ValidFailString_ReturnsOutput(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'

        under_test = multi_test_framework.Result.Fail(mock_test_spec, mock_output, mock_log_output)
        assert mock_output in str(under_test)
        assert mock_log_output in str(under_test)


class TestResultCrash(unittest.TestCase):

    def test_Create_ValidArgs_CorrectAttributes(self):
        mock_test_spec = mock.MagicMock()
        mock_output = mock.MagicMock()
        mock_return_code = mock.MagicMock()
        mock_stacktrace = mock.MagicMock()
        mock_log_output = 'mock_log_output'

        under_test = multi_test_framework.Result.Crash(
            mock_test_spec, mock_output, mock_return_code, mock_stacktrace, mock_log_output)

        assert under_test.test_spec == mock_test_spec
        assert under_test.output == mock_output
        assert under_test.log_output == mock_log_output
        assert under_test.ret_code == mock_return_code
        assert under_test.stacktrace == mock_stacktrace

    def test_Str_ValidCrashString_ReturnsOutput(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'
        mock_return_code = 0
        mock_stacktrace = 'mock stacktrace'

        under_test = multi_test_framework.Result.Crash(
            mock_test_spec, mock_output, mock_return_code, mock_stacktrace, mock_log_output)

        assert mock_stacktrace in str(under_test)
        assert mock_output in str(under_test)
        assert str(mock_return_code) in str(under_test)

    def test_Str_MissingStackTrace_ReturnsCorrectly(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'
        mock_return_code = 0
        mock_stacktrace = None

        under_test = multi_test_framework.Result.Crash(
            mock_test_spec, mock_output, mock_return_code, mock_stacktrace, mock_log_output)

        assert mock_output in str(under_test)
        assert str(mock_return_code) in str(under_test)


class TestResultTimeout(unittest.TestCase):

    def test_Create_ValidArgs_CorrectAttributes(self):
        mock_test_spec = mock.MagicMock()
        mock_output = mock.MagicMock()
        mock_log_output = mock.MagicMock()
        mock_timeout = mock.MagicMock()

        under_test = multi_test_framework.Result.Timeout(mock_test_spec, mock_output, mock_timeout, mock_log_output)
        assert under_test.test_spec == mock_test_spec
        assert under_test.output == mock_output
        assert under_test.log_output == mock_log_output
        assert under_test.time_secs == mock_timeout

    def test_Str_ValidTimeoutString_ReturnsOutput(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'
        mock_timeout = 0

        under_test = multi_test_framework.Result.Timeout(mock_test_spec, mock_output, mock_timeout, mock_log_output)
        assert mock_output in str(under_test)
        assert str(mock_timeout) in str(under_test)


class TestResultUnknown(unittest.TestCase):

    def test_Create_ValidArgs_CorrectAttributes(self):
        mock_test_spec = mock.MagicMock()
        mock_output = mock.MagicMock()
        mock_log_output = mock.MagicMock()
        mock_extra_info = mock.MagicMock()

        under_test = multi_test_framework.Result.Unknown(mock_test_spec, mock_output, mock_extra_info, mock_log_output)

        assert under_test.test_spec == mock_test_spec
        assert under_test.output == mock_output
        assert under_test.log_output == mock_log_output
        assert under_test.extra_info == mock_extra_info

    def test_Str_ValidUnknownString_ReturnsOutput(self):
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_extra_info = 'mock extra info'
        mock_log_output = 'mock_log_output'

        under_test = multi_test_framework.Result.Unknown(mock_test_spec, mock_output, mock_extra_info, mock_log_output)

        assert mock_output in str(under_test)
        assert mock_extra_info in str(under_test)


class TestMultiTestSuite(unittest.TestCase):

    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    def test_TestData_ValidAP_TeardownAPOnce(self, mock_kill_processes):
        mock_abstract_test_suite = multi_test_framework.MultiTestSuite()
        mock_test_data_generator = mock_abstract_test_suite._collected_test_data(mock.MagicMock())
        mock_asset_processor = mock.MagicMock()
        for test_data in mock_test_data_generator:
            test_data.asset_processor = mock_asset_processor
        mock_asset_processor.teardown.assert_called()
        assert test_data.asset_processor is None
        mock_kill_processes.assert_called_once_with(include_asset_processor=True)

    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    def test_TestData_NoAP_NoTeardownAP(self, mock_kill_processes):
        mock_abstract_test_suite = multi_test_framework.MultiTestSuite()
        mock_test_data_generator = mock_abstract_test_suite._collected_test_data(mock.MagicMock())
        for test_data in mock_test_data_generator:
            test_data.asset_processor = None
        mock_kill_processes.assert_called_once_with(include_asset_processor=False)

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite.filter_session_shared_tests')
    def test_PytestCustomModifyItems_FunctionsMatch_AddsRunners(self, mock_filter_tests):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
            pass
        mock_func_1 = mock.MagicMock()
        mock_test = mock.MagicMock()
        runner_1 = multi_test_framework.MultiTestSuite.Runner('mock_runner_1', mock_func_1, [mock_test])
        mock_run_pytest_func = mock.MagicMock()
        runner_1.run_pytestfunc = mock_run_pytest_func
        mock_result_pytestfuncs = [mock.MagicMock()]
        runner_1.result_pytestfuncs = mock_result_pytestfuncs
        mock_items = []
        mock_items.extend(mock_result_pytestfuncs)

        MockTestSuite._runners = [runner_1]
        mock_test_1 = mock.MagicMock()
        mock_test_2 = mock.MagicMock()
        mock_filter_tests.return_value = [mock_test_1, mock_test_2]

        MockTestSuite.pytest_custom_modify_items(mock.MagicMock(), mock_items, mock.MagicMock())
        assert mock_items == [mock_run_pytest_func, mock_result_pytestfuncs[0]]

    def test_GetSingleTests_NoSingleTests_EmptyList(self):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
            pass
        mock_test_suite = MockTestSuite()
        tests = mock_test_suite.get_single_tests()
        assert len(tests) == 0

    def test_GetSingleTests_OneSingleTests_ReturnsOne(self):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
            class MockSingleTest(multi_test_framework.SingleTest):
                pass
        mock_test_suite = MockTestSuite()
        tests = mock_test_suite.get_single_tests()
        assert len(tests) == 1
        assert tests[0].__name__ == "MockSingleTest"
        assert issubclass(tests[0], multi_test_framework.SingleTest)

    def test_GetSingleTests_AllTests_ReturnsOnlySingles(self):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
            class MockSingleTest(multi_test_framework.SingleTest):
                pass
            class MockAnotherSingleTest(multi_test_framework.SingleTest):
                pass
            class MockNotSingleTest(multi_test_framework.SharedTest):
                pass
        mock_test_suite = MockTestSuite()
        tests = mock_test_suite.get_single_tests()
        assert len(tests) == 2
        for test in tests:
            assert issubclass(test, multi_test_framework.SingleTest)

    def test_GetSharedTests_NoSharedTests_EmptyList(self):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
                pass
        mock_test_suite = MockTestSuite()
        tests = mock_test_suite.get_shared_tests()
        assert len(tests) == 0

    def test_GetSharedTests_OneSharedTests_ReturnsOne(self):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
            class MockSharedTest(multi_test_framework.SharedTest):
                pass
        mock_test_suite = MockTestSuite()
        tests = mock_test_suite.get_shared_tests()
        assert len(tests) == 1
        assert tests[0].__name__ == 'MockSharedTest'
        assert issubclass(tests[0], multi_test_framework.SharedTest)

    def test_GetSharedTests_AllTests_ReturnsOnlyShared(self):
        class MockTestSuite(multi_test_framework.MultiTestSuite):
            class MockSharedTest(multi_test_framework.SharedTest):
                pass
            class MockAnotherSharedTest(multi_test_framework.SharedTest):
                pass
            class MockNotSharedTest(multi_test_framework.SingleTest):
                pass
        mock_test_suite = MockTestSuite()
        tests = mock_test_suite.get_shared_tests()
        assert len(tests) == 2
        for test in tests:
            assert issubclass(test, multi_test_framework.SharedTest)

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite.filter_session_shared_tests')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite.get_shared_tests')
    def test_GetSessionSharedTests_Valid_CallsCorrectly(self, mock_get_shared_tests, mock_filter_session):
        multi_test_framework.MultiTestSuite.get_session_shared_tests(mock.MagicMock())
        assert mock_get_shared_tests.called
        assert mock_filter_session.called

    @mock.patch('ly_test_tools.o3de.multi_test_framework.skip_pytest_runtest_setup', mock.MagicMock())
    def test_FilterSessionSharedTests_OneSharedTest_ReturnsOne(self):
        def mock_test():
            pass
        mock_test.originalname = 'mock_test'
        mock_test.__name__ = mock_test.originalname
        mock_session_items = [mock_test]
        mock_shared_tests = [mock_test]

        selected_tests = multi_test_framework.MultiTestSuite.filter_session_shared_tests(mock_session_items, mock_shared_tests)
        assert selected_tests == mock_session_items
        assert len(selected_tests) == 1

    @mock.patch('ly_test_tools.o3de.multi_test_framework.skip_pytest_runtest_setup', mock.MagicMock())
    def test_FilterSessionSharedTests_ManyTests_ReturnsCorrectTests(self):
        def mock_test():
            pass
        def mock_test_2():
            pass
        def mock_test_3():
            pass
        mock_test.originalname = 'mock_test'
        mock_test.__name__ = mock_test.originalname
        mock_test_2.originalname = 'mock_test_2'
        mock_test_2.__name__ = mock_test_2.originalname
        mock_test_3.originalname = 'mock_test_3'
        mock_test_3.__name__ = mock_test_3.originalname
        mock_session_items = [mock_test, mock_test_2]
        mock_shared_tests = [mock_test, mock_test_2, mock_test_3]

        selected_tests = multi_test_framework.MultiTestSuite.filter_session_shared_tests(mock_session_items, mock_shared_tests)
        assert selected_tests == mock_session_items

    @mock.patch('ly_test_tools.o3de.multi_test_framework.skip_pytest_runtest_setup')
    def test_FilterSessionSharedTests_SkipOneTest_ReturnsCorrectTests(self, mock_skip):
        def mock_test():
            pass
        def mock_test_2():
            pass
        def mock_test_3():
            pass
        mock_skip.side_effect = [True, Exception]
        mock_test.originalname = 'mock_test'
        mock_test.__name__ = mock_test.originalname
        mock_test_2.originalname = 'mock_test_2'
        mock_test_2.__name__ = mock_test_2.originalname
        mock_test_3.originalname = 'mock_test_3'
        mock_test_3.__name__ = mock_test_3.originalname
        mock_session_items = [mock_test, mock_test_2]
        mock_shared_tests = [mock_test, mock_test_2, mock_test_3]

        selected_tests = multi_test_framework.MultiTestSuite.filter_session_shared_tests(mock_session_items, mock_shared_tests)
        assert selected_tests == [mock_test]

    @mock.patch('ly_test_tools.o3de.multi_test_framework.skip_pytest_runtest_setup', mock.MagicMock(side_effect=Exception))
    def test_FilterSessionSharedTests_ExceptionDuringSkipSetup_SkipsAddingTest(self):
        def mock_test():
            pass
        mock_test.originalname = 'mock_test'
        mock_test.__name__ = mock_test.originalname
        mock_session_items = [mock_test]
        mock_shared_tests = [mock_test]

        selected_tests = multi_test_framework.MultiTestSuite.filter_session_shared_tests(mock_session_items, mock_shared_tests)
        assert len(selected_tests) == 0

    def test_FilterSharedTests_TrueParams_ReturnsTrueTests(self):
        mock_test = mock.MagicMock()
        mock_test.is_batchable = True
        mock_test.is_parallelizable = True
        mock_test_2 = mock.MagicMock()
        mock_test_2.is_batchable = False
        mock_test_2.is_parallelizable = False
        mock_shared_tests = [mock_test, mock_test_2]

        filtered_tests = multi_test_framework.MultiTestSuite.filter_shared_tests(
            mock_shared_tests, is_batchable=True, is_parallelizable=True)
        assert filtered_tests == [mock_test]

    def test_FilterSharedTests_FalseParams_ReturnsFalseTests(self):
        mock_test = mock.MagicMock()
        mock_test.is_batchable = True
        mock_test.is_parallelizable = True
        mock_test_2 = mock.MagicMock()
        mock_test_2.is_batchable = False
        mock_test_2.is_parallelizable = False
        mock_shared_tests = [mock_test, mock_test_2]

        filtered_tests = multi_test_framework.MultiTestSuite.filter_shared_tests(
            mock_shared_tests, is_batchable=False, is_parallelizable=False)
        assert filtered_tests == [mock_test_2]


class TestUtils(unittest.TestCase):

    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    def test_PrepareAssetProcessor_APExists_StartsAP(self, mock_kill_processes):
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_ap = mock.MagicMock()
        mock_collected_test_data.asset_processor = mock_ap

        prepare_asset_processor(mock_workspace, mock_collected_test_data)
        assert mock_collected_test_data.asset_processor.start.called
        assert not mock_kill_processes.called

    @mock.patch('ly_test_tools.o3de.asset_processor.AssetProcessor.start')
    @mock.patch('ly_test_tools.environment.process_utils.process_exists')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    def test_PrepareAssetProcessor_NoAP_KillAndCreateAP(self, mock_kill_processes, mock_proc_exists, mock_start):
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_collected_test_data.asset_processor = None
        mock_proc_exists.return_value = False

        prepare_asset_processor(mock_workspace, mock_collected_test_data)
        mock_kill_processes.assert_called_with(include_asset_processor=True)
        assert isinstance(mock_collected_test_data.asset_processor, ly_test_tools.o3de.asset_processor.AssetProcessor)
        assert mock_start.called

    @mock.patch('ly_test_tools.o3de.asset_processor.AssetProcessor.start')
    @mock.patch('ly_test_tools.environment.process_utils.process_exists')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    def test_PrepareAssetProcessor_NoAPButProcExists_NoKill(self, mock_kill_processes, mock_proc_exists, mock_start):
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_collected_test_data.asset_processor = None
        mock_proc_exists.return_value = True

        prepare_asset_processor(mock_workspace, mock_collected_test_data)
        mock_kill_processes.assert_called_with(include_asset_processor=False)
        assert not mock_start.called
        assert mock_collected_test_data.asset_processor is None

    @mock.patch('ly_test_tools.o3de.asset_processor.AssetProcessor.start')
    @mock.patch('ly_test_tools.environment.process_utils.process_exists')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    def test_PrepareAssetProcessor_NoAPButProcExists_NoKill(self, mock_kill_processes, mock_proc_exists, mock_start):
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_collected_test_data.asset_processor = None
        mock_proc_exists.return_value = True

        prepare_asset_processor(mock_workspace, mock_collected_test_data)
        mock_kill_processes.assert_called_with(include_asset_processor=False)
        assert not mock_start.called
        assert mock_collected_test_data.asset_processor is None

    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_module_filename')
    def test_GetResultsUsingOutput_ValidJsonSuccess_CreatesPassResult(self, mock_get_module):
        mock_get_module.return_value = 'mock_module_name'
        mock_test_suite = multi_test_framework.MultiTestSuite()
        mock_test = mock.MagicMock()
        mock_test.__name__ = 'mock_test_name'
        mock_test_list = [mock_test]
        mock_output = 'JSON_START(' \
                      '{"name": "mock_module_name", "output": "mock_std_out", "success": "mock_success_data"}' \
                      ')JSON_END'

        results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, '')
        assert len(results) == 1
        assert 'mock_test_name' in results.keys()
        assert isinstance(results['mock_test_name'], multi_test_framework.Result.Pass)

    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_module_filename')
    def test_GetResultsUsingOutput_ValidJsonFail_CreatesFailResult(self, mock_get_module):
        mock_get_module.return_value = 'mock_module_name'
        mock_test_suite = multi_test_framework.MultiTestSuite()
        mock_test = mock.MagicMock()
        mock_test.__name__ = 'mock_test_name'
        mock_test_list = [mock_test]
        mock_output = 'JSON_START(' \
                      '{"name": "mock_module_name", "output": "mock_std_out", "failed": "mock_fail_data", "success": ""}' \
                      ')JSON_END'

        results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, '')
        assert len(results) == 1
        assert 'mock_test_name' in results.keys()
        assert isinstance(results['mock_test_name'], multi_test_framework.Result.Fail)

    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_module_filename')
    def test_GetResultsUsingOutput_ModuleNotInLog_CreatesUnknownResult(self, mock_get_module):
        mock_get_module.return_value = 'different_module_name'
        mock_test_suite = multi_test_framework.MultiTestSuite()
        mock_test = mock.MagicMock()
        mock_test.__name__ = 'mock_test_name'
        mock_test_list = [mock_test]
        mock_output = 'JSON_START(' \
                      '{"name": "mock_module_name", "output": "mock_std_out", "failed": "mock_fail_data"}' \
                      ')JSON_END'

        results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, '')
        assert len(results) == 1
        assert 'mock_test_name' in results.keys()
        assert isinstance(results['mock_test_name'], multi_test_framework.Result.Unknown)

    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_module_filename')
    def test_GetResultsUsingOutput_MultipleTests_CreatesCorrectResults(self, mock_get_module):
        mock_get_module.side_effect = ['mock_module_name_pass', 'mock_module_name_fail', 'different_module_name']
        mock_test_suite = multi_test_framework.MultiTestSuite()
        mock_test_pass = mock.MagicMock()
        mock_test_pass.__name__ = 'mock_test_name_pass'
        mock_test_fail = mock.MagicMock()
        mock_test_fail.__name__ = 'mock_test_name_fail'
        mock_test_unknown = mock.MagicMock()
        mock_test_unknown.__name__ = 'mock_test_name_unknown'
        mock_test_list = [mock_test_pass, mock_test_fail, mock_test_unknown]
        mock_output = 'JSON_START(' \
                      '{"name": "mock_module_name_pass", "output": "mock_std_out", "success": "mock_success_data"}' \
                      ')JSON_END' \
                      'JSON_START(' \
                      '{"name": "mock_module_name_fail", "output": "mock_std_out", "failed": "mock_fail_data", "success": ""}' \
                      ')JSON_END' \
                      'JSON_START(' \
                      '{"name": "mock_module_name_unknown", "output": "mock_std_out", "failed": "mock_fail_data", "success": ""}' \
                      ')JSON_END'
        mock_log_output = 'JSON_START(' \
                          '{"name": "mock_module_name_pass"}' \
                          ')JSON_END' \
                          'JSON_START(' \
                          '{"name": "mock_module_name_fail"}' \
                          ')JSON_END'

        results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, mock_log_output)
        assert len(results) == 3
        assert 'mock_test_name_pass' in results.keys()
        assert 'mock_test_name_fail' in results.keys()
        assert 'mock_test_name_unknown' in results.keys()
        assert isinstance(results['mock_test_name_pass'], multi_test_framework.Result.Pass)
        assert isinstance(results['mock_test_name_fail'], multi_test_framework.Result.Fail)
        assert isinstance(results['mock_test_name_unknown'], multi_test_framework.Result.Unknown)

    @mock.patch('builtins.print')
    def test_ReportResult_TestPassed_ReportsCorrectly(self, mock_print):
        mock_test_name = 'mock name'
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'
        mock_pass = ly_test_tools.o3de.multi_test_framework.Result.Pass(mock_test_spec, mock_output, mock_log_output)
        ly_test_tools.o3de.multi_test_framework.MultiTestSuite._report_result(mock_test_name, mock_pass)
        mock_print.assert_called_with(
            f'Test {mock_test_name}:\nTest Passed\n------------\n'
            f'|  Output  |\n------------\n{mock_output}\n')

    @mock.patch('pytest.fail')
    def test_ReportResult_TestFailed_FailsCorrectly(self, mock_pytest_fail):
        mock_test_name = 'mock name'
        mock_test_spec = mock.MagicMock()
        mock_output = 'mock_output'
        mock_log_output = 'mock_log_output'
        mock_fail = ly_test_tools.o3de.multi_test_framework.Result.Fail(mock_test_spec, mock_output, mock_log_output)

        ly_test_tools.o3de.multi_test_framework.MultiTestSuite._report_result(mock_test_name, mock_fail)
        mock_pytest_fail.assert_called_with(
            f"Test {mock_test_name}:\n"
            "Test FAILED\n"
            "------------\n"
            f"|  Output  |\n"
            "------------\n"
            f"{mock_output}\n"
            "----------------------------------------------------\n"
            f"| Application log  |\n"
            "----------------------------------------------------\n"
            f"{mock_log_output}\n")


class TestRunningTests(unittest.TestCase):

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.path.join', mock.MagicMock())
    def test_ExecSingleTest_TestSucceeds_ReturnsPass(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                     mock_retrieve_log, mock_retrieve_editor_log,
                                                     mock_get_output_results):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_executable.get_returncode.return_value = 0
        mock_get_output_results.return_value = {}

        results = mock_test_suite._exec_single_test(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec, [])
        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Pass)
        assert mock_cycle_crash.called
        assert mock_executable.start.called

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.path.join', mock.MagicMock())
    def test_ExecSingleTest_TestFails_ReturnsFail(self, mock_cycle_crash, mock_get_testcase_filepath, mock_retrieve_log,
                                                  mock_retrieve_editor_log, mock_get_output_results):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_executable.get_returncode.return_value = 15
        mock_get_output_results.return_value = {}

        results = mock_test_suite._exec_single_test(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec, [])

        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Fail)
        assert mock_cycle_crash.called
        assert mock_executable.start.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_crash_output')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.path.join', mock.MagicMock())
    @mock.patch('os.path.basename', mock.MagicMock())
    def test_ExecSingleTest_TestCrashes_ReturnsCrash(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                     mock_retrieve_log, mock_retrieve_editor_log,
                                                     mock_get_output_results, mock_retrieve_crash):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_executable.get_returncode.return_value = 1
        mock_get_output_results.return_value = {}

        results = mock_test_suite._exec_single_test(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec, [])
        assert mock_cycle_crash.call_count == 2
        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Crash)
        assert mock_executable.start.called
        assert mock_retrieve_crash.called
        # Save executable log output, crash log, and crash dmp
        assert mock_workspace.artifact_manager.save_artifact.call_count == 3

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    def test_ExecSingleTest_TestTimeout_ReturnsTimeout(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                       mock_retrieve_log, mock_retrieve_editor_log,
                                                       mock_get_output_results):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_executable.wait.side_effect = ly_test_tools.launchers.exceptions.WaitTimeoutError()
        mock_get_output_results.return_value = {}

        results = mock_test_suite._exec_single_test(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec, [])

        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Timeout)
        assert mock_cycle_crash.called
        assert mock_executable.start.called
        assert mock_executable.stop.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.unlink', mock.MagicMock())
    @mock.patch('tempfile.NamedTemporaryFile', mock.MagicMock())
    @mock.patch('os.path.join', mock.MagicMock())
    @mock.patch('os.path.splitext', mock.MagicMock())
    @mock.patch('os.path.dirname', mock.MagicMock())
    def test_ExecMultitest_AllTestsPass_ReturnsPasses(self, mock_cycle_crash):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_artifact_manager = mock.MagicMock()
        mock_workspace.artifact_manager = mock_artifact_manager
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_executable.args = []
        mock_executable.get_returncode.return_value = 0
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_test_spec_2 = mock.MagicMock()
        mock_test_spec_2.__name__ = 'mock_test_name_2'
        mock_test_spec_list = [mock_test_spec, mock_test_spec_2]
        mock_path_exists = mock.MagicMock()
        mock_path_exists.return_value = True

        with mock.patch('builtins.open', mock.mock_open(read_data="")) as mock_file:
            results = mock_test_suite._exec_multitest(
                mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec_list, [])

        assert len(results) == 2
        assert isinstance(results['mock_test_name'], multi_test_framework.Result.Pass)
        assert isinstance(results['mock_test_name_2'], multi_test_framework.Result.Pass)
        assert mock_cycle_crash.called

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.unlink', mock.MagicMock())
    @mock.patch('tempfile.NamedTemporaryFile', mock.MagicMock())
    @mock.patch('os.path.join', mock.MagicMock())
    @mock.patch('os.path.splitext', mock.MagicMock())
    @mock.patch('os.path.dirname', mock.MagicMock())
    def test_ExecMultitest_OneFailure_CallsCorrectFunc(
            self, mock_cycle_crash, mock_get_testcase_filepath, mock_get_results):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_executable.get_returncode.return_value = 15
        mock_test_spec = mock.MagicMock()
        mock_test_spec_2 = mock.MagicMock()
        mock_test_spec_list = [mock_test_spec, mock_test_spec_2]
        mock_get_testcase_filepath.side_effect = ['mock_path', 'mock_path_2']
        mock_get_results.return_value = {'mock_test_name': mock.MagicMock(), 'mock_test_name_2': mock.MagicMock()}

        with mock.patch('builtins.open', mock.mock_open(read_data="")) as mock_file:
            results = mock_test_suite._exec_multitest(
                mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec_list, [])

        assert len(results) == 2
        assert mock_cycle_crash.called
        assert mock_get_results.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_crash_output')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.unlink', mock.MagicMock())
    @mock.patch('tempfile.NamedTemporaryFile', mock.MagicMock())
    @mock.patch('os.path.join', mock.MagicMock())
    @mock.patch('os.path.basename', mock.MagicMock())
    @mock.patch('os.path.dirname', mock.MagicMock())
    def test_ExecMultitest_OneCrash_ReportsOnUnknownResult(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                           mock_retrieve_log, mock_retrieve_editor_log,
                                                           mock_get_results, mock_retrieve_crash):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_executable.get_returncode.return_value = 1
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_test_spec_2 = mock.MagicMock()
        mock_test_spec_2.__name__ = 'mock_test_name_2'
        mock_test_spec_list = [mock_test_spec, mock_test_spec_2]
        mock_unknown_result = ly_test_tools.o3de.multi_test_framework.Result.Unknown(test_spec=None)
        mock_unknown_result.test_spec = mock.MagicMock()
        mock_unknown_result.editor_log = mock.MagicMock()
        mock_get_testcase_filepath.side_effect = ['mock_path', 'mock_path_2']
        mock_get_results.return_value = {mock_test_spec.__name__: mock_unknown_result,
                                         mock_test_spec_2.__name__: mock.MagicMock()}

        results = mock_test_suite._exec_multitest(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec_list, [])
        assert mock_cycle_crash.call_count == 2
        assert mock_get_results.called
        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Crash)
        # Save executable log output, crash log, and crash dmp
        assert mock_workspace.artifact_manager.save_artifact.call_count == 3

    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_crash_output')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    @mock.patch('os.unlink', mock.MagicMock())
    @mock.patch('tempfile.NamedTemporaryFile', mock.MagicMock())
    @mock.patch('os.path.join', mock.MagicMock())
    @mock.patch('os.path.basename', mock.MagicMock())
    @mock.patch('os.path.dirname', mock.MagicMock())
    def test_ExecMultitest_ManyUnknown_ReportsUnknownResults(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                             mock_retrieve_log, mock_retrieve_editor_log,
                                                             mock_get_results, mock_retrieve_crash):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_executable.get_returncode.return_value = 1
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_test_spec_2 = mock.MagicMock()
        mock_test_spec_2.__name__ = 'mock_test_name_2'
        mock_test_spec_list = [mock_test_spec, mock_test_spec_2]
        mock_unknown_result = ly_test_tools.o3de.multi_test_framework.Result.Unknown(test_spec=None)
        mock_unknown_result.__name__ = 'mock_test_name'
        mock_unknown_result.test_spec = mock.MagicMock()
        mock_unknown_result.test_spec.__name__ = 'mock_test_spec_name'
        mock_unknown_result.editor_log = mock.MagicMock()
        mock_get_testcase_filepath.side_effect = ['mock_path', 'mock_path_2']
        mock_get_results.return_value = {mock_test_spec.__name__: mock_unknown_result,
                                         mock_test_spec_2.__name__: mock_unknown_result}

        results = mock_test_suite._exec_multitest(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec_list, [])
        assert mock_cycle_crash.call_count == 2
        assert mock_get_results.called
        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Crash)
        assert isinstance(results[mock_test_spec_2.__name__], multi_test_framework.Result.Unknown)
        assert results[mock_test_spec_2.__name__].extra_info, "Extra info missing from Unknown failure"

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_results_using_output')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_editor_log_content')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
    def test_ExecMultitest_EditorTimeout_ReportsCorrectly(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                          mock_retrieve_log, mock_retrieve_editor_log, mock_get_results):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_workspace = mock.MagicMock()
        mock_workspace.paths.engine_root.return_value = ""
        mock_executable = mock.MagicMock()
        mock_executable.wait.side_effect = ly_test_tools.launchers.exceptions.WaitTimeoutError()
        mock_test_spec = mock.MagicMock()
        mock_test_spec.__name__ = 'mock_test_name'
        mock_test_spec_2 = mock.MagicMock()
        mock_test_spec_2.__name__ = 'mock_test_name_2'
        mock_test_spec_list = [mock_test_spec, mock_test_spec_2]
        mock_unknown_result = ly_test_tools.o3de.multi_test_framework.Result.Unknown(test_spec=None)
        mock_unknown_result.test_spec = mock.MagicMock()
        mock_unknown_result.test_spec.__name__ = 'mock_test_spec_name'
        mock_unknown_result.output = mock.MagicMock()
        mock_unknown_result.editor_log = mock.MagicMock()
        mock_get_testcase_filepath.side_effect = ['mock_path', 'mock_path_2']
        mock_get_results.return_value = {mock_test_spec.__name__: mock_unknown_result,
                                         mock_test_spec_2.__name__: mock_unknown_result}

        results = mock_test_suite._exec_multitest(
            mock.MagicMock(), mock_workspace, mock_executable, 0, 'mock_log_name', mock_test_spec_list, [])
        assert mock_cycle_crash.called
        assert mock_get_results.called
        assert isinstance(results[mock_test_spec.__name__], multi_test_framework.Result.Timeout)
        assert isinstance(results[mock_test_spec_2.__name__], multi_test_framework.Result.Unknown)
        assert results[mock_test_spec_2.__name__].extra_info, "Extra info missing from Unknown failure"

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._report_result')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._exec_single_test')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunSingleTest_ValidTest_ReportsResults(self, mock_exec_single_test, mock_report_result):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_test_spec = mock.MagicMock()
        mock_result = mock.MagicMock()
        mock_test_name = 'mock_test_result'
        mock_exec_single_test.return_value = {mock_test_name: mock_result}

        mock_test_suite._run_single_test(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec)

        assert mock_exec_single_test.called
        assert mock_collected_test_data.results.update.called
        mock_report_result.assert_called_with(mock_test_name, mock_result)

    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._test_reporting')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._exec_multitest')
    def test_RunBatchedTests_ValidTests_CallsCorrectly(self, mock_exec_multitest, mock_reporting):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_test_suite.atom_tools_executable_name = "dummy_executable"
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_test_spec_list = mock.MagicMock()
        mock_extra_cmdline_args = [""]

        mock_test_suite._run_batched_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_exec_multitest.called
        assert mock_reporting.called
        assert type(mock_test_suite.executable) in [WinAtomToolsLauncher, LinuxAtomToolsLauncher]
        assert mock_test_suite.executable.workspace == mock_workspace

    @mock.patch('threading.Thread')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_number_parallel_executables')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunParallelTests_TwoTestsAndExecutables_TwoThreads(self, mock_get_number_executables, mock_thread):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_extra_cmdline_args = [""]
        mock_get_number_executables.return_value = 2
        mock_test_spec_list = [mock.MagicMock(), mock.MagicMock()]

        mock_test_suite._run_parallel_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_collected_test_data.results.update.call_count == len(mock_test_spec_list)
        assert mock_thread.call_count == len(mock_test_spec_list)

    @mock.patch('threading.Thread')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_number_parallel_executables')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunParallelTests_TenTestsAndTwoExecutables_TenThreads(self, mock_get_number_executables, mock_thread):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_extra_cmdline_args = [""]
        mock_get_number_executables.return_value = 2
        mock_test_spec_list = []
        for i in range(10):
            mock_test_spec_list.append(mock.MagicMock())

        mock_test_suite._run_parallel_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_collected_test_data.results.update.call_count == len(mock_test_spec_list)
        assert mock_thread.call_count == len(mock_test_spec_list)

    @mock.patch('threading.Thread')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_number_parallel_executables')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunParallelTests_TenTestsAndThreeEditors_TenThreads(self, mock_get_number_executables, mock_thread):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_extra_cmdline_args = [""]
        mock_get_number_executables.return_value = 3
        mock_test_spec_list = []
        for i in range(10):
            mock_test_spec_list.append(mock.MagicMock())

        mock_test_suite._run_parallel_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_collected_test_data.results.update.call_count == len(mock_test_spec_list)
        assert mock_thread.call_count == len(mock_test_spec_list)

    @mock.patch('threading.Thread')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_number_parallel_executables')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunParallelBatchedTests_TwoTestsAndEditors_TwoThreads(self, mock_get_number_executables, mock_thread):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_extra_cmdline_args = [""]
        mock_get_number_executables.return_value = 2
        mock_test_spec_list = [mock.MagicMock(), mock.MagicMock()]

        mock_test_suite._run_parallel_batched_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_collected_test_data.results.update.call_count == len(mock_test_spec_list)
        assert mock_thread.call_count == len(mock_test_spec_list)

    @mock.patch('threading.Thread')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_number_parallel_executables')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunParallelBatchedTests_TenTestsAndTwoEditors_2Threads(self, mock_get_number_executables, mock_thread):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_extra_cmdline_args = [""]
        mock_get_number_executables.return_value = 2
        mock_test_spec_list = []
        for i in range(10):
            mock_test_spec_list.append(mock.MagicMock())

        mock_test_suite._run_parallel_batched_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_collected_test_data.results.update.call_count == 2
        assert mock_thread.call_count == 2

    @mock.patch('threading.Thread')
    @mock.patch('ly_test_tools.o3de.multi_test_framework.MultiTestSuite._get_number_parallel_executables')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
    def test_RunParallelBatchedTests_TenTestsAndThreeEditors_ThreeThreads(
            self, mock_get_number_executables, mock_thread):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()
        mock_extra_cmdline_args = [""]
        mock_get_number_executables.return_value = 3
        mock_test_spec_list = []
        for i in range(10):
            mock_test_spec_list.append(mock.MagicMock())

        mock_test_suite._run_parallel_batched_tests(
            mock_request, mock_workspace, mock_collected_test_data, mock_test_spec_list, mock_extra_cmdline_args)

        assert mock_collected_test_data.results.update.call_count == 3
        assert mock_thread.call_count == 3

    def test_GetNumberParallelEditors_ConfigExists_ReturnsConfig(self):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_request.config.getoption.return_value = 1

        number_of_executables = mock_test_suite._get_number_parallel_executables(mock_request)
        assert number_of_executables == 1

    def test_GetNumberParallelEditors_ConfigNotExists_ReturnsDefault(self):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_request = mock.MagicMock()
        mock_request.config.getoption.return_value = None

        number_of_executables = mock_test_suite._get_number_parallel_executables(mock_request)
        assert number_of_executables == mock_test_suite.get_number_parallel_executables()

    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.prepare_asset_processor')
    @mock.patch('ly_test_tools.launchers.launcher_helper.create_editor')
    @mock.patch('ly_test_tools.launchers.launcher_helper.create_atom_tools_launcher')
    def test_SetupTest_AtomExe_SetsExecutable(self, mock_create_atom, mock_create_editor, mock_prepare_ap, 
                                              mock_kill_processes):
        mock_workspace = mock.MagicMock()
        mock_exe = 'mock_atom_exe'
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_test_suite.atom_tools_executable_name = mock_exe
        mock_test_suite.executable = mock.MagicMock()

        mock_test_suite._setup_test(mock_workspace, mock.MagicMock())
        mock_create_atom.assert_called_once_with(mock_workspace, mock_exe)
        assert not mock_create_editor.called
        assert mock_prepare_ap.called
        assert mock_kill_processes.called
        assert mock_test_suite.executable.configure_settings.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.kill_all_ly_processes')
    @mock.patch('ly_test_tools.o3de.editor_test_utils.prepare_asset_processor')
    @mock.patch('ly_test_tools.launchers.launcher_helper.create_editor')
    @mock.patch('ly_test_tools.launchers.launcher_helper.create_atom_tools_launcher')
    def test_SetupTest_EditorExe_SetsExecutable(self, mock_create_atom, mock_create_editor, mock_prepare_ap, 
                                                mock_kill_processes):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_test_suite.atom_tools_executable_name = None
        mock_test_suite.executable = mock.MagicMock()

        mock_test_suite._setup_test(mock.MagicMock(), mock.MagicMock())
        assert not mock_create_atom.called
        assert mock_create_editor.called
        assert mock_prepare_ap.called
        assert mock_kill_processes.called
        assert mock_test_suite.executable.configure_settings.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs')
    def test_TestReporting_AllPassResults_NoSaveLogs(self, mock_save_logs):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_collected_test_data = mock.MagicMock()
        mock_pass = multi_test_framework.Result.Pass(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
        mock_results = [mock_pass]

        mock_test_suite._test_reporting(mock_collected_test_data, mock_results, mock.MagicMock(), mock.MagicMock())
        assert not mock_save_logs.called
        assert mock_collected_test_data.results.update.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs')
    def test_TestReporting_UnknownResults_SaveLogs(self, mock_save_logs):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_collected_test_data = mock.MagicMock()
        mock_results = [None]

        mock_test_suite._test_reporting(mock_collected_test_data, mock_results, mock.MagicMock(), mock.MagicMock())
        assert mock_save_logs.called
        assert mock_collected_test_data.results.update.called

    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs')
    def test_TestReporting_BothResults_SaveLogs(self, mock_save_logs):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_collected_test_data = mock.MagicMock()
        mock_pass = multi_test_framework.Result.Pass(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
        mock_results = [mock_pass, None]

        mock_test_suite._test_reporting(mock_collected_test_data, mock_results, mock.MagicMock(), mock.MagicMock())
        assert mock_save_logs.called
        assert mock_collected_test_data.results.update.called
        
    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs')
    def test_TestReporting_ManyResults_SaveLogs(self, mock_save_logs):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_collected_test_data = mock.MagicMock()
        mock_pass = multi_test_framework.Result.Pass(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
        mock_fail = multi_test_framework.Result.Fail(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
        mock_results = [mock_pass, mock_fail, None]

        mock_test_suite._test_reporting(mock_collected_test_data, mock_results, mock.MagicMock(), mock.MagicMock())
        assert mock_save_logs.called
        assert mock_collected_test_data.results.update.called
 
    def test_SetupCmdlineArgs_EditorExe_EnablesPrefab(self):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_executable = WinEditor(mock.MagicMock(), [])
        mock_test_spec_list = mock.MagicMock()

        under_test = mock_test_suite._setup_cmdline_args(None, mock_executable, mock_test_spec_list, mock.MagicMock())
        assert "--regset=/Amazon/Preferences/EnablePrefabSystem=true" in under_test
        assert "-rhi=null" in under_test
        assert "--attach-debugger" not in under_test
        assert "--wait-for-debugger" not in under_test

    def test_SetupCmdlineArgs_AtomExe_NotEnablesPrefab(self):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_executable = WinAtomToolsLauncher(mock.MagicMock(), [])
        mock_test_spec_list = mock.MagicMock()

        under_test = mock_test_suite._setup_cmdline_args(None, mock_executable, mock_test_spec_list, mock.MagicMock())
        assert "--regset=/Amazon/Preferences/EnablePrefabSystem=true" not in under_test
        assert "-rhi=null" in under_test
        assert "--attach-debugger" not in under_test
        assert "--wait-for-debugger" not in under_test

    def test_SetupCmdlineArgs_Debugger_EnablesDebugger(self):
        mock_test_suite = ly_test_tools.o3de.multi_test_framework.MultiTestSuite()
        mock_executable = WinAtomToolsLauncher(mock.MagicMock(), [])
        mock_test_spec = mock.MagicMock()
        mock_test_spec.attach_debugger = True
        mock_test_spec.wait_for_debugger = True
        mock_test_spec_list = [mock_test_spec]

        under_test = mock_test_suite._setup_cmdline_args(None, mock_executable, mock_test_spec_list, mock.MagicMock())
        assert "--regset=/Amazon/Preferences/EnablePrefabSystem=true" not in under_test
        assert "-rhi=null" in under_test
        assert "--attach-debugger" in under_test
        assert "--wait-for-debugger" in under_test

@mock.patch('_pytest.python.Class.collect')
class TestMultiTestCollector(unittest.TestCase):

    def setUp(self):
        mock_name = mock.MagicMock()
        mock_collector = mock.MagicMock()
        self.mock_test_class = ly_test_tools.o3de.multi_test_framework.MultiTestSuite.MultiTestCollector.from_parent(
            name=mock_name, parent=mock_collector)
        self.mock_test_class.obj = mock.MagicMock()
        single_1 = mock.MagicMock()
        single_1.__name__ = 'single_1_name'
        single_2 = mock.MagicMock()
        single_2.__name__ = 'single_2_name'
        self.mock_test_class.obj.get_single_tests.return_value = [single_1, single_2]
        batch_1 = mock.MagicMock()
        batch_1.__name__ = 'batch_1_name'
        batch_2 = mock.MagicMock()
        batch_2.__name__ = 'batch_2_name'
        parallel_1 = mock.MagicMock()
        parallel_1.__name__ = 'parallel_1_name'
        parallel_2 = mock.MagicMock()
        parallel_2.__name__ = 'parallel_2_name'
        both_1 = mock.MagicMock()
        both_1.__name__ = 'both_1_name'
        both_2 = mock.MagicMock()
        both_2.__name__ = 'both_2_name'
        self.mock_test_class.obj.filter_shared_tests.side_effect = [ [batch_1, batch_2],
                                                                     [parallel_1, parallel_2],
                                                                     [both_1, both_2] ]

    def test_Collect_NoParallelNoBatched_RunsAsSingleTests(self, mock_collect):
        self.mock_test_class.config.getoption.return_value = True
        self.mock_test_class.collect()
        assert self.mock_test_class.obj.single_1_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.single_2_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.batch_1_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.batch_2_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.parallel_1_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.parallel_2_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.both_1_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.both_2_name.__name__ == 'single_run'

    def test_Collect_AllValidTests_RunsAsInteded(self, mock_collect):
        self.mock_test_class.config.getoption.return_value = False
        self.mock_test_class.collect()
        assert self.mock_test_class.obj.single_1_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.single_2_name.__name__ == 'single_run'
        assert self.mock_test_class.obj.batch_1_name.__name__ == 'result'
        assert self.mock_test_class.obj.batch_2_name.__name__ == 'result'
        assert self.mock_test_class.obj.parallel_1_name.__name__ == 'result'
        assert self.mock_test_class.obj.parallel_2_name.__name__ == 'result'
        assert self.mock_test_class.obj.both_1_name.__name__ == 'result'
        assert self.mock_test_class.obj.both_2_name.__name__ == 'result'

    def test_Collect_AllValidTests_CallsCollect(self, mock_collect):
        self.mock_test_class.collect()
        assert mock_collect.called

    def test_Collect_NormalCollection_ReturnsFilteredRuns(self, mock_collect):
        mock_run = mock.MagicMock()
        mock_run.obj.marks = {"run_type": 'run_shared'}
        mock_run_2 = mock.MagicMock()
        mock_run_2.obj.marks = {"run_type": 'result'}
        mock_collect.return_value = [mock_run_2]

        collection = self.mock_test_class.collect()
        assert collection == [mock_run_2]

    def test_Collect_NormalRun_ReturnsRunners(self, mock_collect):
        self.mock_test_class.collect()
        runners =  self.mock_test_class.obj._runners

        assert runners[0].name == 'run_batched_tests'
        assert runners[1].name == 'run_parallel_tests'
        assert runners[2].name == 'run_parallel_batched_tests'

    def test_Collect_NormalCollection_StoresRunners(self, mock_collect):
        mock_runner = mock.MagicMock()
        mock_run = mock.MagicMock()
        mock_run.obj.marks = {"run_type": 'run_shared'}
        mock_run.function.marks = {"runner": mock_runner}
        mock_runner_2 = mock.MagicMock()
        mock_runner_2.result_pytestfuncs = []
        mock_run_2 = mock.MagicMock()
        mock_run_2.obj.marks = {"run_type": 'result'}
        mock_run_2.function.marks = {"runner": mock_runner_2}
        mock_collect.return_value = [mock_run_2]

        self.mock_test_class.collect()

        assert mock_run_2 in mock_runner_2.result_pytestfuncs

    @mock.patch('ly_test_tools.o3de.multi_test_framework.isinstance', mock.MagicMock())
    @mock.patch('ly_test_tools.o3de.multi_test_framework.issubclass', mock.MagicMock())
    def test_MakeSingleRun_SingleRun_SetupTeardown(self, mock_collect):
        mock_inner_test_spec = mock.MagicMock()
        mock_self = mock.MagicMock()

        mock_single_run_func = multi_test_framework.MultiTestSuite.MultiTestCollector._make_single_run(
            mock_inner_test_spec)
        mock_single_run_func(mock_self, mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), mock.MagicMock())

        mock_inner_test_spec.setup.assert_called_once()
        mock_inner_test_spec.teardown.assert_called_once()
        mock_self._run_single_test.assert_called_once()

    @mock.patch('ly_test_tools.o3de.multi_test_framework.isinstance', mock.MagicMock(return_value=False))
    @mock.patch('ly_test_tools.o3de.multi_test_framework.issubclass', mock.MagicMock(return_value=False))
    def test_MakeSingleRun_NoSingleRun_NoSetupTeardown(self, mock_collect):
        mock_inner_test_spec = mock.MagicMock()
        mock_self = mock.MagicMock()

        mock_single_run_func = multi_test_framework.MultiTestSuite.MultiTestCollector._make_single_run(
            mock_inner_test_spec)
        mock_single_run_func(mock_self, mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), mock.MagicMock())

        assert not mock_inner_test_spec.setup.called
        assert not mock_inner_test_spec.teardown.called
        mock_self._run_single_test.assert_called_once()

    def test_CreateRunner_HappyPath_SetsRunnersProperly(self, mock_collect):
        mock_runner_name = 'mock_runner_name'
        mock_function = mock.MagicMock()
        mock_shared_test_spec_1 = mock.MagicMock()
        mock_shared_test_spec_1.__name__ = 'mock_shared_test_spec_1'
        mock_shared_test_spec_2 = mock.MagicMock()
        mock_shared_test_spec_2.__name__ = 'mock_shared_test_spec_2'
        mock_tests = [mock_shared_test_spec_1, mock_shared_test_spec_2]
        mock_obj = mock.MagicMock()

        multi_test_framework.MultiTestSuite.MultiTestCollector._create_runner(mock_runner_name, mock_function,
                                                                              mock_tests, mock_obj)

        assert hasattr(mock_obj, mock_runner_name)
        assert hasattr(mock_obj, mock_shared_test_spec_1.__name__)
        assert hasattr(mock_obj, mock_shared_test_spec_2.__name__)

    def test_CreateRunner_MakeSharedRun_CallsAttributeFunc(self, mock_collect):
        mock_runner_name = 'mock_runner_name'
        mock_obj = mock.MagicMock()
        mock_function = mock.MagicMock()
        mock_function.__name__ = 'mock_func_name'
        mock_tests = []
        mock_request = mock.MagicMock()
        mock_workspace = mock.MagicMock()
        mock_collected_test_data = mock.MagicMock()

        multi_test_framework.MultiTestSuite.MultiTestCollector._create_runner(mock_runner_name, mock_function,
                                                                              mock_tests, mock_obj)
        getattr(mock_obj, mock_runner_name)(mock_obj, mock_request, mock_workspace, mock_collected_test_data,
                                            mock.MagicMock())

        mock_obj.mock_func_name.assert_called_once_with(mock_request, mock_workspace, mock_collected_test_data,
                                                        mock_tests)

    def test_CreateRunner_MakeResultsRun_CallsAttributeFunc(self, mock_collect):
        mock_runner_name = 'mock_runner_name'
        mock_obj = mock.MagicMock()
        mock_function = mock.MagicMock()
        mock_shared_test_spec_1 = mock.MagicMock()
        mock_shared_test_spec_1.__name__ = 'mock_shared_test_spec_name_1'
        mock_shared_test_spec_2 = mock.MagicMock()
        mock_shared_test_spec_2.__name__ = 'mock_shared_test_spec_name_2'
        mock_tests = [mock_shared_test_spec_1, mock_shared_test_spec_2]
        mock_collected_test_data = mock.MagicMock()
        mock_result_1 = mock.MagicMock()
        mock_result_2 = mock.MagicMock()
        mock_collected_test_data.results = {mock_shared_test_spec_1.__name__: mock_result_1,
                                            mock_shared_test_spec_2.__name__: mock_result_2}

        multi_test_framework.MultiTestSuite.MultiTestCollector._create_runner(mock_runner_name, mock_function,
                                                                              mock_tests, mock_obj)
        getattr(mock_obj, mock_shared_test_spec_1.__name__)(mock_obj, mock.MagicMock(), mock.MagicMock(),
                                                            mock_collected_test_data, mock.MagicMock())
        mock_obj._report_result.assert_called_with(mock_shared_test_spec_1.__name__, mock_result_1)
        getattr(mock_obj, mock_shared_test_spec_2.__name__)(mock_obj, mock.MagicMock(), mock.MagicMock(),
                                                            mock_collected_test_data, mock.MagicMock())
        mock_obj._report_result.assert_called_with(mock_shared_test_spec_2.__name__, mock_result_2)