Merged PR 104: Disallow ignore/accept hit intrinsics in lib export functions

Disallow ignore/accept hit intrinsics in lib export functions

- still need to add general validator part to restrict intrinsics for
  library functions and shaders.

lib/HLSL/DxilOperations.cpp

@@ -589,7 +589,7 @@ void OP::GetMinShaderModelAndMask(OpCode C, bool bWithTranslation,
   // Instructions: IgnoreHit=155, AcceptHitAndEndSearch=156
   if ((155 <= op && op <= 156)) {
     major = 6;  minor = 3;
-    mask = SFLAG(Library) | SFLAG(AnyHit);
+    mask = SFLAG(AnyHit);
     return;
   }
   // Instructions: CallShader=159

lib/HLSL/DxilValidation.cpp

@@ -802,7 +802,7 @@ static bool ValidateOpcodeInProfile(DXIL::OpCode opcode,
   // Instructions: IgnoreHit=155, AcceptHitAndEndSearch=156
   if ((155 <= op && op <= 156))
     return (pSM->GetMajor() > 6 || (pSM->GetMajor() == 6 && pSM->GetMinor() >= 3))
-        && (pSM->IsLib() || pSM->GetKind() == DXIL::ShaderKind::AnyHit);
+        && (pSM->GetKind() == DXIL::ShaderKind::AnyHit);
   // Instructions: CallShader=159
   if (op == 159)
     return (pSM->GetMajor() > 6 || (pSM->GetMajor() == 6 && pSM->GetMinor() >= 3))

tools/clang/test/CodeGenHLSL/quick-test/raytracing_accept_ignore_hit_fail_lib.hlsl

@@ -0,0 +1,26 @@
+// RUN: %dxc -T lib_6_3 -auto-binding-space 11 %s | FileCheck %s
+
+// TODO: verify failure for exported library function and flip the CHECK below
+// CHECK: define void @"\01?libfunc
+
+struct Payload {
+  float foo;
+};
+struct Attr {
+  float2 bary;
+};
+
+void libfunc(inout Payload p, Attr a)  {
+  float4 result = 2.6;
+  if (p.foo < 2) {
+    result.x = 1;             // unused
+    AcceptHitAndEndSearch();  // does not return
+    result.y = 3;             // dead code
+  }
+  if (a.bary.x < 9) {
+    result.x = 2;             // unused
+    IgnoreHit();              // does not return
+    result.y = 4;             // dead code
+  }
+   p.foo = result;
+}

utils/hct/hctdb.py

@@ -356,7 +356,7 @@ class db_dxil(object):
         for i in "IgnoreHit,AcceptHitAndEndSearch".split(","):
             self.name_idx[i].category = "AnyHit Terminals"
             self.name_idx[i].shader_model = 6,3
-            self.name_idx[i].shader_stages = ("library","anyhit")
+            self.name_idx[i].shader_stages = ("anyhit",)
         for i in "CallShader".split(","):
             self.name_idx[i].category = "Indirect Shader Invocation"
             self.name_idx[i].shader_model = 6,3