Adjust the expected precision for Sin/Cos/Tan tests for 16-bit types. (#1921)

The old test values and precision was the same as the 32-bit variants of the tests. This does not work for 16-bit floating point numbers and especially for trigonometric operations.

tools/clang/unittests/HLSL/ShaderOpArithTable.xml

@@ -1375,7 +1375,7 @@
         </Row>
         <Row Name="TanHalf">
             <Parameter Name="Validation.Type">Epsilon</Parameter>
-            <Parameter Name="Validation.Tolerance">0.0008</Parameter>
+            <Parameter Name="Validation.Tolerance">0.016</Parameter>
             <Parameter Name="ShaderOp.Text"> struct SUnaryFPOp {
                 float16_t input;
                 float16_t output;
@@ -1396,8 +1396,15 @@
                 <Value>0</Value>
                 <Value>denorm</Value>
                 <Value>Inf</Value>
-                <Value>-314</Value>
-                <Value>314</Value>
+                <Value>0.6279297</Value>
+                <Value>1.255859</Value>
+                <Value>1.884766</Value>
+                <Value>2.511719</Value>
+                <Value>3.140625</Value>
+                <Value>3.769531</Value>
+                <Value>4.398438</Value>
+                <Value>5.652344</Value>
+                <Value>6.281250</Value>
             </Parameter>
             <Parameter Name="Validation.Expected1">
                 <Value>NaN</Value>
@@ -1407,8 +1414,15 @@
                 <Value>0</Value>
                 <Value>0</Value>
                 <Value>NaN</Value>
-                <Value>0.1606257</Value>
-                <Value>-0.1606257</Value>
+                <Value>0.72594857</Value>
+                <Value>3.06955433</Value>
+                <Value>-3.07967043</Value>
+                <Value>-0.72892153</Value>
+                <Value>-0.00096773</Value>
+                <Value>0.72596157</Value>
+                <Value>3.07986474</Value>
+                <Value>-0.7304042</Value>
+                <Value>-0.00193546</Value>
             </Parameter>
             <Parameter Name="ShaderOp.Arguments">-enable-16bit-types</Parameter>
         </Row>
@@ -1684,7 +1698,7 @@
         </Row>
         <Row Name="SinHalf">
             <Parameter Name="Validation.Type">Epsilon</Parameter>
-            <Parameter Name="Validation.Tolerance">0.0008</Parameter>
+            <Parameter Name="Validation.Tolerance">0.003</Parameter>
             <Parameter Name="ShaderOp.Text"> struct SUnaryFPOp {
                 float16_t input;
                 float16_t output;
@@ -1705,8 +1719,16 @@
                 <Value>0</Value>
                 <Value>denorm</Value>
                 <Value>Inf</Value>
-                <Value>-314</Value>
-                <Value>314</Value>
+                <Value>0.6279297</Value>
+                <Value>1.255859</Value>
+                <Value>1.884766</Value>
+                <Value>2.511719</Value>
+                <Value>3.140625</Value>
+                <Value>3.769531</Value>
+                <Value>4.398438</Value>
+                <Value>5.023438</Value>
+                <Value>5.652344</Value>
+                <Value>6.281250</Value>
             </Parameter>
             <Parameter Name="Validation.Expected1">
                 <Value>NaN</Value>
@@ -1716,8 +1738,16 @@
                 <Value>0</Value>
                 <Value>0</Value>
                 <Value>NaN</Value>
-                <Value>0.1585929</Value>
-                <Value>-0.1585929</Value>
+                <Value>0.58747065</Value>
+                <Value>0.95081574</Value>
+                <Value>0.95111507</Value>
+                <Value>0.58904284</Value>
+                <Value>0.00096773</Value>
+                <Value>-0.58747751</Value>
+                <Value>-0.95112079</Value>
+                <Value>-0.95201313</Value>
+                <Value>-0.58982444</Value>
+                <Value>-0.00193545</Value>
             </Parameter>
             <Parameter Name="ShaderOp.Arguments">-enable-16bit-types</Parameter>
         </Row>
@@ -1975,7 +2005,7 @@
         </Row>
         <Row Name="CosHalf">
             <Parameter Name="Validation.Type">Epsilon</Parameter>
-            <Parameter Name="Validation.Tolerance">0.0008</Parameter>
+            <Parameter Name="Validation.Tolerance">0.003</Parameter>
             <Parameter Name="ShaderOp.Text"> struct SUnaryFPOp {
                 float16_t input;
                 float16_t output;
@@ -1996,8 +2026,16 @@
                 <Value>0</Value>
                 <Value>denorm</Value>
                 <Value>Inf</Value>
-                <Value>-314</Value>
-                <Value>314</Value>
+                <Value>0.6279297</Value>
+                <Value>1.255859</Value>
+                <Value>1.884766</Value>
+                <Value>2.511719</Value>
+                <Value>3.140625</Value>
+                <Value>3.769531</Value>
+                <Value>4.398438</Value>
+                <Value>5.023438</Value>
+                <Value>5.652344</Value>
+                <Value>6.281250</Value>
             </Parameter>
             <Parameter Name="Validation.Expected1">
                 <Value>NaN</Value>
@@ -2007,8 +2045,16 @@
                 <Value>1.0</Value>
                 <Value>1.0</Value>
                 <Value>NaN</Value>
-                <Value>0.987344</Value>
-                <Value>0.987344</Value>
+                <Value>0.80924553</Value>
+                <Value>0.30975693</Value>
+                <Value>-0.30883664</Value>
+                <Value>-0.80810183</Value>
+                <Value>-0.99999952</Value>
+                <Value>-0.80924052</Value>
+                <Value>-0.30881903</Value>
+                <Value>0.30605716</Value>
+                <Value>0.80753154</Value>
+                <Value>0.99999809</Value>
             </Parameter>
             <Parameter Name="ShaderOp.Arguments">-enable-16bit-types</Parameter>
         </Row>

utils/hct/hctdb_test.py

@@ -568,12 +568,14 @@ def add_test_cases():
     ]], [[
         'NaN', 'NaN', '-0', '-0', '0', '0', 'NaN', '-0.0007346401',
         '0.0007346401'
-    ]], "unary float", "sin", half_inputs=[[
-        'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', '-314',
-        '314'
+    ]], "unary float", "sin", half_validation_tolerance=0.003, half_inputs=[[
+        'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', 
+        '0.6279297', '1.255859', '1.884766', '2.511719', '3.140625',
+        '3.769531', '4.398438', '5.023438', '5.652344', '6.281250'
     ]], half_outputs=[[
-        'NaN', 'NaN', '-0', '-0', '0', '0', 'NaN', '0.1585929',
-        '-0.1585929'
+        'NaN', 'NaN', '-0', '-0', '0', '0', 'NaN', 
+        '0.58747065', '0.95081574', '0.95111507', '0.58904284', '0.00096773',
+        '-0.58747751', '-0.95112079', '-0.95201313', '-0.58982444', '-0.00193545'
     ]])
     add_test_case_float_half('Cos', ['Cos'], 'Epsilon', 0.0008, [[
         'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', '-314.16',
@@ -581,12 +583,14 @@ def add_test_cases():
     ]], [[
         'NaN', 'NaN', '1.0', '1.0', '1.0', '1.0', 'NaN', '0.99999973015',
         '0.99999973015'
-    ]], "unary float", "cos", half_inputs=[[
-        'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', '-314',
-        '314'
+    ]], "unary float", "cos", half_validation_tolerance=0.003, half_inputs=[[
+        'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', 
+        '0.6279297', '1.255859', '1.884766', '2.511719', '3.140625',
+        '3.769531', '4.398438', '5.023438', '5.652344', '6.281250'
     ]], half_outputs=[[
-        'NaN', 'NaN', '1.0', '1.0', '1.0', '1.0', 'NaN', '0.987344',
-        '0.987344'
+        'NaN', 'NaN', '1.0', '1.0', '1.0', '1.0', 'NaN', 
+        '0.80924553', '0.30975693', '-0.30883664', '-0.80810183', '-0.99999952',
+        '-0.80924052', '-0.30881903', '0.30605716', '0.80753154', '0.99999809'
     ]])
     add_test_case_float_half('Tan', ['Tan'], 'Epsilon', 0.0008, [[
         'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', '-314.16',
@@ -594,12 +598,14 @@ def add_test_cases():
     ]], [[
         'NaN', 'NaN', '-0.0', '-0.0', '0.0', '0.0', 'NaN', '-0.000735',
         '0.000735'
-    ]], "unary float", "tan", half_inputs=[[
-        'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', '-314',
-        '314'
+    ]], "unary float", "tan", half_validation_tolerance=0.016, half_inputs=[[
+        'NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', 
+        '0.6279297', '1.255859', '1.884766', '2.511719', '3.140625',
+        '3.769531', '4.398438', '5.652344', '6.281250'
     ]], half_outputs=[[
-        'NaN', 'NaN', '-0', '-0', '0', '0', 'NaN', '0.1606257',
-        '-0.1606257'
+        'NaN', 'NaN', '-0', '-0', '0', '0', 'NaN', 
+        '0.72594857', '3.06955433', '-3.07967043', '-0.72892153', '-0.00096773', 
+        '0.72596157', '3.07986474', '-0.7304042', '-0.00193546'
     ]])
     add_test_case_float_half('Hcos', ['Hcos'], 'Epsilon', 0.0008,
         [['NaN', '-Inf', '-denorm', '-0', '0', 'denorm', 'Inf', '1', '-1']], [[