Browse Source

Warn if frameworks exceed the maximum number of test mutations (#9667)

> The number of test mutations will be limited to 10.
https://github.com/TechEmpower/FrameworkBenchmarks/issues/8420

This change shows a warning message for frameworks if the maximum is reached.
Petrik de Heus 1 tháng trước cách đây
mục cha
commit
f4188f0039
1 tập tin đã thay đổi với 12 bổ sung0 xóa
  1. 12 0
      toolset/utils/metadata.py

+ 12 - 0
toolset/utils/metadata.py

@@ -191,6 +191,18 @@ class Metadata:
                     % config['framework'],
                     color=Fore.YELLOW)
 
+            # Check that each framework does not have more than the maximum number of tests
+            maximum_tests = 10
+            non_broken_tests_filter = lambda test: (not hasattr(test, "tags")) or ("broken" not in test.tags)
+            non_broken_tests_to_run = list(filter(non_broken_tests_filter, tests_to_run))
+            if len(non_broken_tests_to_run) > maximum_tests:
+                message = [
+                    "Framework %s defines %s tests in benchmark_config.json (max is %s)."
+                        % (config['framework'], len(non_broken_tests_to_run), maximum_tests),
+                    "Contact maintainers and remove deprecated or discarded ones to make room."
+                ]
+                log("\n".join(message), color=Fore.YELLOW)
+
             # Check that each test configuration is acceptable
             # Throw exceptions if a field is missing, or how to improve the field
             for test_name, test_keys in test.items():