[spirv] Improve codegen for NonUniform (#2884)

* [spirv] Move NonUniform decoration processing to a separate IMR.

* [spirv] Propagate NonUniform through atomic operations.

* [spirv] Perform OpCopyObject before marking non-uniform.

* [spirv] Add comments about NonUniform decoration handling.

tools/clang/include/clang/SPIRV/SpirvBuilder.h

@@ -165,6 +165,10 @@ public:
   SpirvLoad *createLoad(const SpirvType *resultType, SpirvInstruction *pointer,
                         SourceLocation loc);
 
+  /// \brief Creates an OpCopyObject instruction from the given pointer.
+  SpirvCopyObject *createCopyObject(QualType resultType,
+                                    SpirvInstruction *pointer, SourceLocation);
+
   /// \brief Creates a store instruction storing the given value into the given
   /// address.
   void createStore(SpirvInstruction *address, SpirvInstruction *value,

tools/clang/include/clang/SPIRV/SpirvInstruction.h

@@ -82,17 +82,20 @@ public:
     // Normal instruction kinds
     // In alphabetical order
 
-    IK_AccessChain,        // OpAccessChain
-    IK_Atomic,             // OpAtomic*
-    IK_Barrier,            // Op*Barrier
-    IK_BinaryOp,           // Binary operations
-    IK_BitFieldExtract,    // OpBitFieldExtract
-    IK_BitFieldInsert,     // OpBitFieldInsert
-    IK_CompositeConstruct, // OpCompositeConstruct
-    IK_CompositeExtract,   // OpCompositeExtract
-    IK_CompositeInsert,    // OpCompositeInsert
-    IK_ExtInst,            // OpExtInst
-    IK_FunctionCall,       // OpFunctionCall
+    IK_AccessChain,                 // OpAccessChain
+    IK_ArrayLength,                 // OpArrayLength
+    IK_Atomic,                      // OpAtomic*
+    IK_Barrier,                     // Op*Barrier
+    IK_BinaryOp,                    // Binary operations
+    IK_BitFieldExtract,             // OpBitFieldExtract
+    IK_BitFieldInsert,              // OpBitFieldInsert
+    IK_CompositeConstruct,          // OpCompositeConstruct
+    IK_CompositeExtract,            // OpCompositeExtract
+    IK_CompositeInsert,             // OpCompositeInsert
+    IK_CopyObject,                  // OpCopyObject
+    IK_DemoteToHelperInvocationEXT, // OpDemoteToHelperInvocationEXT
+    IK_ExtInst,                     // OpExtInst
+    IK_FunctionCall,                // OpFunctionCall
 
     IK_EndPrimitive, // OpEndPrimitive
     IK_EmitVertex,   // OpEmitVertex
@@ -103,22 +106,20 @@ public:
     IK_GroupNonUniformElect,    // OpGroupNonUniformElect
     IK_GroupNonUniformUnaryOp,  // Group non-uniform unary operations
 
-    IK_ImageOp,                     // OpImage*
-    IK_ImageQuery,                  // OpImageQuery*
-    IK_ImageSparseTexelsResident,   // OpImageSparseTexelsResident
-    IK_ImageTexelPointer,           // OpImageTexelPointer
-    IK_Load,                        // OpLoad
-    IK_SampledImage,                // OpSampledImage
-    IK_Select,                      // OpSelect
-    IK_SpecConstantBinaryOp,        // SpecConstant binary operations
-    IK_SpecConstantUnaryOp,         // SpecConstant unary operations
-    IK_Store,                       // OpStore
-    IK_UnaryOp,                     // Unary operations
-    IK_VectorShuffle,               // OpVectorShuffle
-    IK_ArrayLength,                 // OpArrayLength
-    IK_RayTracingOpNV,              // NV raytracing ops
-    IK_DemoteToHelperInvocationEXT, // OpDemoteToHelperInvocationEXT
-    IK_RayQueryOpKHR,               // KHR rayquery ops
+    IK_ImageOp,                   // OpImage*
+    IK_ImageQuery,                // OpImageQuery*
+    IK_ImageSparseTexelsResident, // OpImageSparseTexelsResident
+    IK_ImageTexelPointer,         // OpImageTexelPointer
+    IK_Load,                      // OpLoad
+    IK_RayQueryOpKHR,             // KHR rayquery ops
+    IK_RayTracingOpNV,            // NV raytracing ops
+    IK_SampledImage,              // OpSampledImage
+    IK_Select,                    // OpSelect
+    IK_SpecConstantBinaryOp,      // SpecConstant binary operations
+    IK_SpecConstantUnaryOp,       // SpecConstant unary operations
+    IK_Store,                     // OpStore
+    IK_UnaryOp,                   // Unary operations
+    IK_VectorShuffle,             // OpVectorShuffle
   };
 
   virtual ~SpirvInstruction() = default;
@@ -1508,6 +1509,25 @@ private:
   llvm::Optional<spv::MemoryAccessMask> memoryAccess;
 };
 
+/// \brief OpCopyObject instruction
+class SpirvCopyObject : public SpirvInstruction {
+public:
+  SpirvCopyObject(QualType resultType, SourceLocation loc,
+                  SpirvInstruction *pointer);
+
+  // For LLVM-style RTTI
+  static bool classof(const SpirvInstruction *inst) {
+    return inst->getKind() == IK_CopyObject;
+  }
+
+  bool invokeVisitor(Visitor *v) override;
+
+  SpirvInstruction *getPointer() const { return pointer; }
+
+private:
+  SpirvInstruction *pointer;
+};
+
 /// \brief OpSampledImage instruction
 /// Result Type must be the OpTypeSampledImage type whose Image Type operand is
 /// the type of Image. We store the QualType for the underlying image as result

tools/clang/include/clang/SPIRV/SpirvVisitor.h

@@ -104,6 +104,7 @@ public:
   DEFINE_VISIT_METHOD(SpirvImageSparseTexelsResident)
   DEFINE_VISIT_METHOD(SpirvImageTexelPointer)
   DEFINE_VISIT_METHOD(SpirvLoad)
+  DEFINE_VISIT_METHOD(SpirvCopyObject)
   DEFINE_VISIT_METHOD(SpirvSampledImage)
   DEFINE_VISIT_METHOD(SpirvSelect)
   DEFINE_VISIT_METHOD(SpirvSpecConstantBinaryOp)

tools/clang/lib/SPIRV/CMakeLists.txt

@@ -15,6 +15,7 @@ add_clang_library(clangSPIRV
   InitListHandler.cpp
   LiteralTypeVisitor.cpp
   LowerTypeVisitor.cpp
+  NonUniformVisitor.cpp
   PreciseVisitor.cpp
   RawBufferMethods.cpp
   RelaxedPrecisionVisitor.cpp

tools/clang/lib/SPIRV/EmitVisitor.cpp

@@ -1005,6 +1005,17 @@ bool EmitVisitor::visit(SpirvLoad *inst) {
   return true;
 }
 
+bool EmitVisitor::visit(SpirvCopyObject *inst) {
+  initInstruction(inst);
+  curInst.push_back(inst->getResultTypeId());
+  curInst.push_back(getOrAssignResultId<SpirvInstruction>(inst));
+  curInst.push_back(getOrAssignResultId<SpirvInstruction>(inst->getPointer()));
+  finalizeInstruction();
+  emitDebugNameForInstruction(getOrAssignResultId<SpirvInstruction>(inst),
+                              inst->getDebugName());
+  return true;
+}
+
 bool EmitVisitor::visit(SpirvSampledImage *inst) {
   initInstruction(inst);
   curInst.push_back(inst->getResultTypeId());

tools/clang/lib/SPIRV/EmitVisitor.h

@@ -249,6 +249,7 @@ public:
   bool visit(SpirvImageSparseTexelsResident *);
   bool visit(SpirvImageTexelPointer *);
   bool visit(SpirvLoad *);
+  bool visit(SpirvCopyObject *);
   bool visit(SpirvSampledImage *);
   bool visit(SpirvSelect *);
   bool visit(SpirvSpecConstantBinaryOp *);

tools/clang/lib/SPIRV/NonUniformVisitor.cpp

@@ -0,0 +1,61 @@
+//===--- NonUniformVisitor.cpp - NonUniform Visitor --------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "NonUniformVisitor.h"
+
+namespace clang {
+namespace spirv {
+
+bool NonUniformVisitor::visit(SpirvLoad *instr) {
+  if(instr->getPointer()->isNonUniform())
+    instr->setNonUniform();
+  return true;
+}
+
+bool NonUniformVisitor::visit(SpirvAccessChain *instr) {
+  bool isNonUniform = instr->isNonUniform() || instr->getBase()->isNonUniform();
+  for (auto *index : instr->getIndexes())
+    isNonUniform = isNonUniform || index->isNonUniform();
+  instr->setNonUniform(isNonUniform);
+  return true;
+}
+
+bool NonUniformVisitor::visit(SpirvUnaryOp *instr) {
+  if (instr->getOperand()->isNonUniform())
+    instr->setNonUniform();
+  return true;
+}
+
+bool NonUniformVisitor::visit(SpirvBinaryOp *instr) {
+  if (instr->getOperand1()->isNonUniform() ||
+      instr->getOperand2()->isNonUniform())
+    instr->setNonUniform();
+  return true;
+}
+
+bool NonUniformVisitor::visit(SpirvSampledImage *instr) {
+  if (instr->getImage()->isNonUniform() || instr->getSampler()->isNonUniform())
+    instr->setNonUniform();
+  return true;
+}
+
+bool NonUniformVisitor::visit(SpirvImageTexelPointer *instr) {
+  if (instr->getImage()->isNonUniform())
+    instr->setNonUniform();
+  return true;
+}
+
+bool NonUniformVisitor::visit(SpirvAtomic *instr) {
+  if (instr->getPointer()->isNonUniform())
+    instr->setNonUniform();
+  return true;
+}
+
+} // end namespace spirv
+} // end namespace clang

tools/clang/lib/SPIRV/NonUniformVisitor.h

@@ -0,0 +1,56 @@
+//===--- NonUniformVisitor.h - NonUniform Visitor ----------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_SPIRV_NONUNIFORMVISITOR_H
+#define LLVM_CLANG_LIB_SPIRV_NONUNIFORMVISITOR_H
+
+#include "clang/SPIRV/FeatureManager.h"
+#include "clang/SPIRV/SpirvContext.h"
+#include "clang/SPIRV/SpirvVisitor.h"
+
+namespace clang {
+namespace spirv {
+
+class SpirvBuilder;
+
+/// Propagates the NonUniform decoration. According to the Vulkan Spec:
+///
+/// If an instruction loads from or stores to a resource (including atomics and
+/// image instructions) and the resource descriptor being accessed is not
+/// dynamically uniform, then the operand corresponding to that resource (e.g.
+/// the pointer or sampled image operand) must be decorated with NonUniformEXT.
+///
+class NonUniformVisitor : public Visitor {
+public:
+  NonUniformVisitor(SpirvContext &spvCtx, const SpirvCodeGenOptions &opts)
+      : Visitor(opts, spvCtx) {}
+
+
+  bool visit(SpirvLoad *);
+  bool visit(SpirvAccessChain *);
+  bool visit(SpirvUnaryOp *);
+  bool visit(SpirvBinaryOp *);
+  bool visit(SpirvSampledImage *);
+  bool visit(SpirvImageTexelPointer *);
+  bool visit(SpirvAtomic *);
+
+  /// The "sink" visit function for all instructions.
+  ///
+  /// By default, all other visit instructions redirect to this visit function.
+  /// So that you want override this visit function to handle all instructions,
+  /// regardless of their polymorphism.
+  bool visitInstruction(SpirvInstruction *instr) { return true; }
+
+private:
+};
+
+} // end namespace spirv
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_SPIRV_NONUNIFORMVISITOR_H

tools/clang/lib/SPIRV/SpirvBuilder.cpp

@@ -12,6 +12,7 @@
 #include "EmitVisitor.h"
 #include "LiteralTypeVisitor.h"
 #include "LowerTypeVisitor.h"
+#include "NonUniformVisitor.h"
 #include "PreciseVisitor.h"
 #include "RelaxedPrecisionVisitor.h"
 #include "RemoveBufferBlockVisitor.h"
@@ -163,7 +164,6 @@ SpirvLoad *SpirvBuilder::createLoad(QualType resultType,
   auto *instruction = new (context) SpirvLoad(resultType, loc, pointer);
   instruction->setStorageClass(pointer->getStorageClass());
   instruction->setLayoutRule(pointer->getLayoutRule());
-  instruction->setNonUniform(pointer->isNonUniform());
   instruction->setRValue(true);
 
   if (pointer->containsAliasComponent() &&
@@ -180,6 +180,19 @@ SpirvLoad *SpirvBuilder::createLoad(QualType resultType,
   return instruction;
 }
 
+SpirvCopyObject *SpirvBuilder::createCopyObject(QualType resultType,
+                                                SpirvInstruction *pointer,
+                                                SourceLocation loc) {
+  assert(insertPoint && "null insert point");
+  auto *instruction = new (context) SpirvCopyObject(resultType, loc, pointer);
+  instruction->setStorageClass(pointer->getStorageClass());
+  instruction->setLayoutRule(pointer->getLayoutRule());
+  // The result of OpCopyObject is always an rvalue.
+  instruction->setRValue(true);
+  insertPoint->addInstruction(instruction);
+  return instruction;
+}
+
 SpirvLoad *SpirvBuilder::createLoad(const SpirvType *resultType,
                                     SpirvInstruction *pointer,
                                     SourceLocation loc) {
@@ -199,7 +212,6 @@ SpirvLoad *SpirvBuilder::createLoad(const SpirvType *resultType,
   }
 
   instruction->setLayoutRule(pointer->getLayoutRule());
-  instruction->setNonUniform(pointer->isNonUniform());
   instruction->setRValue(true);
   insertPoint->addInstruction(instruction);
   return instruction;
@@ -245,10 +257,6 @@ SpirvBuilder::createAccessChain(QualType resultType, SpirvInstruction *base,
       new (context) SpirvAccessChain(resultType, loc, base, indexes);
   instruction->setStorageClass(base->getStorageClass());
   instruction->setLayoutRule(base->getLayoutRule());
-  bool isNonUniform = base->isNonUniform();
-  for (auto *index : indexes)
-    isNonUniform = isNonUniform || index->isNonUniform();
-  instruction->setNonUniform(isNonUniform);
   instruction->setContainsAliasComponent(base->containsAliasComponent());
 
   // If doing an access chain into a structured or byte address buffer, make
@@ -266,7 +274,6 @@ SpirvUnaryOp *SpirvBuilder::createUnaryOp(spv::Op op, QualType resultType,
                                           SourceLocation loc) {
   assert(insertPoint && "null insert point");
   auto *instruction = new (context) SpirvUnaryOp(op, resultType, loc, operand);
-  instruction->setNonUniform(operand->isNonUniform());
   insertPoint->addInstruction(instruction);
   return instruction;
 }
@@ -278,7 +285,6 @@ SpirvBinaryOp *SpirvBuilder::createBinaryOp(spv::Op op, QualType resultType,
   assert(insertPoint && "null insert point");
   auto *instruction =
       new (context) SpirvBinaryOp(op, resultType, loc, lhs, rhs);
-  instruction->setNonUniform(lhs->isNonUniform() || rhs->isNonUniform());
   insertPoint->addInstruction(instruction);
   return instruction;
 }
@@ -357,7 +363,6 @@ SpirvSampledImage *SpirvBuilder::createSampledImage(QualType imageType,
   assert(insertPoint && "null insert point");
   auto *sampledImage =
       new (context) SpirvSampledImage(imageType, loc, image, sampler);
-  sampledImage->setNonUniform(image->isNonUniform() || sampler->isNonUniform());
   insertPoint->addInstruction(sampledImage);
   return sampledImage;
 }
@@ -368,7 +373,6 @@ SpirvImageTexelPointer *SpirvBuilder::createImageTexelPointer(
   assert(insertPoint && "null insert point");
   auto *instruction = new (context)
       SpirvImageTexelPointer(resultType, loc, image, coordinate, sample);
-  instruction->setNonUniform(image->isNonUniform());
   insertPoint->addInstruction(instruction);
   return instruction;
 }
@@ -1080,11 +1084,15 @@ std::vector<uint32_t> SpirvBuilder::takeModule() {
   CapabilityVisitor capabilityVisitor(astContext, context, spirvOptions, *this);
   RelaxedPrecisionVisitor relaxedPrecisionVisitor(context, spirvOptions);
   PreciseVisitor preciseVisitor(context, spirvOptions);
+  NonUniformVisitor nonUniformVisitor(context, spirvOptions);
   RemoveBufferBlockVisitor removeBufferBlockVisitor(context, spirvOptions);
   EmitVisitor emitVisitor(astContext, context, spirvOptions);
 
   mod->invokeVisitor(&literalTypeVisitor, true);
 
+  // Propagate NonUniform decorations
+  mod->invokeVisitor(&nonUniformVisitor);
+
   // Lower types
   mod->invokeVisitor(&lowerTypeVisitor);
 

tools/clang/lib/SPIRV/SpirvEmitter.cpp

@@ -7504,15 +7504,17 @@ SpirvEmitter::processIntrinsicNonUniformResourceIndex(const CallExpr *expr) {
   auto *index = doExpr(expr->getArg(0));
   // Decorate the expression in NonUniformResourceIndex() with NonUniformEXT.
   // Aside from this, we also need to eventually populate the NonUniformEXT
-  // status to the usage of this expression: the "pointer" operand to a memory
-  // access instruction. Vulkan spec has the following rules:
+  // status to the usages of this expression. This is done by the
+  // NonUniformVisitor class.
   //
-  // If an instruction loads from or stores to a resource (including atomics and
-  // image instructions) and the resource descriptor being accessed is not
-  // dynamically uniform, then the operand corresponding to that resource (e.g.
-  // the pointer or sampled image operand) must be decorated with NonUniformEXT.
-  index->setNonUniform();
-  return index;
+  // The decoration shouldn't be applied to the operand, rather to a copy of the
+  // result. Even though applying the decoration to the operand may not be
+  // functionally incorrect (since adding NonUniform is more conservative), it
+  // could affect performance and isn't the intent of the shader.
+  auto *copyInstr =
+      spvBuilder.createCopyObject(expr->getType(), index, expr->getExprLoc());
+  copyInstr->setNonUniform();
+  return copyInstr;
 }
 
 SpirvInstruction *

tools/clang/lib/SPIRV/SpirvInstruction.cpp

@@ -72,6 +72,7 @@ DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvImageQuery)
 DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvImageSparseTexelsResident)
 DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvImageTexelPointer)
 DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvLoad)
+DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvCopyObject)
 DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvSampledImage)
 DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvSelect)
 DEFINE_INVOKE_VISITOR_FOR_CLASS(SpirvSpecConstantBinaryOp)
@@ -692,6 +693,11 @@ SpirvLoad::SpirvLoad(QualType resultType, SourceLocation loc,
     : SpirvInstruction(IK_Load, spv::Op::OpLoad, resultType, loc),
       pointer(pointerInst), memoryAccess(mask) {}
 
+SpirvCopyObject::SpirvCopyObject(QualType resultType, SourceLocation loc,
+                                 SpirvInstruction *pointerInst)
+    : SpirvInstruction(IK_CopyObject, spv::Op::OpCopyObject, resultType, loc),
+      pointer(pointerInst) {}
+
 SpirvSampledImage::SpirvSampledImage(QualType resultType, SourceLocation loc,
                                      SpirvInstruction *imageInst,
                                      SpirvInstruction *samplerInst)

tools/clang/test/CodeGenSPIRV/intrinsics.non-uniform-resource-index.hlsl

@@ -62,6 +62,7 @@
 // CHECK: OpDecorate [[nu50:%\d+]] NonUniform
 // CHECK: OpDecorate [[nu51:%\d+]] NonUniform
 // CHECK: OpDecorate [[nu52:%\d+]] NonUniform
+// CHECK: OpDecorate [[nu53:%\d+]] NonUniform
 
 Texture2D           gTextures[32];
 SamplerState        gSamplers[];
@@ -71,101 +72,118 @@ RWBuffer<uint>      gRWBuffers[32];
 SubpassInput        gSubpassInputs[32];
 
 float4 main(uint index : A, float2 loc : B, int2 offset : C) : SV_Target {
-// CHECK: [[nu1]] = OpLoad %uint %index
-// CHECK: [[nu2]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu1]]
-// CHECK: [[nu3]] = OpLoad %type_2d_image
-// CHECK: [[nu4]] = OpIAdd %uint {{%\d+}} %uint_1
-// CHECK: [[nu5]] = OpAccessChain %_ptr_UniformConstant_type_sampler %gSamplers [[nu4]]
-// CHECK: [[nu6]] = OpLoad %type_sampler
-// CHECK: [[nu7]] = OpSampledImage %type_sampled_image
-// CHECK:           OpImageSampleImplicitLod
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:    [[nu1]] = OpCopyObject %uint [[i]]
+// CHECK:    [[nu2]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu1]]
+// CHECK:    [[nu3]] = OpLoad %type_2d_image
+// CHECK: [[i:%\d+]] = OpIAdd %uint {{%\d+}} %uint_1
+// CHECK:    [[nu4]] = OpCopyObject %uint [[i]]
+// CHECK:    [[nu5]] = OpAccessChain %_ptr_UniformConstant_type_sampler %gSamplers [[nu4]]
+// CHECK:    [[nu6]] = OpLoad %type_sampler
+// CHECK:    [[nu7]] = OpSampledImage %type_sampled_image
+// CHECK:              OpImageSampleImplicitLod
     float4 v1 = gTextures[NonUniformResourceIndex(index)].Sample(
         gSamplers[NonUniformResourceIndex(index + 1)], loc);
 
-// CHECK:  [[nu8]] = OpLoad %uint %index
-// CHECK:            OpIAdd %uint {{%\d+}} %uint_1
-// CHECK:  [[nu9]] = OpAccessChain %_ptr_UniformConstant_type_sampler %gSamplers [[nu8]]
-// CHECK: [[nu10]] = OpLoad %type_sampler
-// CHECK: [[nu11]] = OpSampledImage %type_sampled_image
-// CHECK:            OpImageSampleImplicitLod
+// CHECK:      [[i:%\d+]] = OpLoad %uint %index
+// CHECK: [[iplus1:%\d+]] = OpIAdd %uint [[i]] %uint_1
+// CHECK:                   OpStore %index [[iplus1]]
+// CHECK:         [[nu8]] = OpCopyObject %uint [[i]]
+// CHECK:         [[nu9]] = OpAccessChain %_ptr_UniformConstant_type_sampler %gSamplers [[nu8]]
+// CHECK:        [[nu10]] = OpLoad %type_sampler
+// CHECK:        [[nu11]] = OpSampledImage %type_sampled_image
+// CHECK:                   OpImageSampleImplicitLod
     float4 v2 = gTextures[0].Sample(
         gSamplers[NonUniformResourceIndex(index++)], loc);
 
-// CHECK:            OpLoad %uint %index
-// CHECK:            OpISub %uint {{%\d+}} %uint_1
-// CHECK: [[nu12]] = OpLoad %uint %index
-// CHECK: [[nu13]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu12]]
-// CHECK: [[nu14]] = OpLoad %type_2d_image
-// CHECK: [[nu15]] = OpSampledImage %type_sampled_image
-// CHECK:            OpImageSampleImplicitLod
+// CHECK:       [[i:%\d+]] = OpLoad %uint %index
+// CHECK: [[iminus1:%\d+]] = OpISub %uint [[i]] %uint_1
+// CHECK:                    OpStore %index [[iminus1]]
+// CHECK:       [[i:%\d+]] = OpLoad %uint %index
+// CHECK:         [[nu12]] = OpCopyObject %uint [[i]]
+// CHECK:         [[nu13]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu12]]
+// CHECK:         [[nu14]] = OpLoad %type_2d_image
+// CHECK:         [[nu15]] = OpSampledImage %type_sampled_image
+// CHECK:                    OpImageSampleImplicitLod
     float4 v3 = gTextures[NonUniformResourceIndex(--index)].Sample(
         gSamplers[0], loc);
 
-// CHECK: [[nu16]] = OpIMul %uint
-// CHECK: [[nu17]] = OpAccessChain %_ptr_UniformConstant_type_2d_image_0 %gRWTextures [[nu16]]
-// CHECK: [[nu18]] = OpLoad %type_2d_image_0
-// CHECK:            OpImageRead
+// CHECK: [[mul:%\d+]] = OpIMul %uint
+// CHECK:     [[nu16]] = OpCopyObject %uint [[mul]]
+// CHECK:     [[nu17]] = OpAccessChain %_ptr_UniformConstant_type_2d_image_0 %gRWTextures [[nu16]]
+// CHECK:     [[nu18]] = OpLoad %type_2d_image_0
+// CHECK:                OpImageRead
     float4 v4 = gRWTextures[NonUniformResourceIndex(index * index)].Load(loc);
 
-// CHECK: [[nu19]] = OpLoad %uint %index
+// CHECK:[[i:%\d+]] = OpLoad %uint %index
+// CHECK: [[nu19]] = OpCopyObject %uint [[i]]
 // CHECK: [[nu20]] = OpUMod %uint [[nu19]] %uint_3
 // CHECK: [[nu21]] = OpAccessChain %_ptr_UniformConstant_type_2d_image_0 %gRWTextures [[nu20]]
 // CHECK: [[nu22]] = OpLoad %type_2d_image_0
 // CHECK:            OpImageWrite
     gRWTextures[NonUniformResourceIndex(index) % 3][loc] = 4;
 
-// CHECK: [[nu23]] = OpLoad %uint %index
-// CHECK:            OpLoad %uint %index
-// CHECK: [[nu24]] = OpIMul %uint [[nu23]] {{%\d+}}
-// CHECK: [[nu25]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image %gBuffers [[nu24]]
-// CHECK: [[nu26]] = OpLoad %type_buffer_image
-// CHECK:            OpImageFetch
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu23]] = OpCopyObject %uint [[i]]
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu24]] = OpIMul %uint [[nu23]] [[i]]
+// CHECK:   [[nu25]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image %gBuffers [[nu24]]
+// CHECK:   [[nu26]] = OpLoad %type_buffer_image
+// CHECK:              OpImageFetch
     float4 v5 = gBuffers[NonUniformResourceIndex(index) * index][5];
 
-// CHECK: [[nu27]] = OpLoad %uint %index
-// CHECK: [[nu28]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image_0 %gRWBuffers [[nu27]]
-// CHECK: [[nu29]] = OpLoad %type_buffer_image_0
-// CHECK:            OpImageRead
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu27]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu28]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image_0 %gRWBuffers [[nu27]]
+// CHECK:   [[nu29]] = OpLoad %type_buffer_image_0
+// CHECK:              OpImageRead
     float4 v6 = gRWBuffers[NonUniformResourceIndex(index)].Load(6);
 
-// CHECK: [[nu30]] = OpLoad %uint %index
-// CHECK: [[nu31]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image_0 %gRWBuffers [[nu30]]
-// CHECK: [[nu32]] = OpLoad %type_buffer_image_0
-// CHECK:            OpImageWrite
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu30]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu31]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image_0 %gRWBuffers [[nu30]]
+// CHECK:   [[nu32]] = OpLoad %type_buffer_image_0
+// CHECK:              OpImageWrite
     gRWBuffers[NonUniformResourceIndex(index)][8] = 9;
 
-// CHECK: [[nu33]] = OpLoad %uint %index
-// CHECK: [[nu34]] = OpLoad %uint %index
-// CHECK: [[nu35]] = OpIMul %uint [[nu33]] [[nu34]]
-// CHECK: [[nu36]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image_0 %gRWBuffers [[nu35]]
-// CHECK: [[nu37]] = OpImageTexelPointer %_ptr_Image_uint {{%\d+}} %uint_10 %uint_0
-// CHECK:            OpAtomicIAdd
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu33]] = OpCopyObject %uint [[i]]
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu34]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu35]] = OpIMul %uint [[nu33]] [[nu34]]
+// CHECK:   [[nu36]] = OpAccessChain %_ptr_UniformConstant_type_buffer_image_0 %gRWBuffers [[nu35]]
+// CHECK:   [[nu37]] = OpImageTexelPointer %_ptr_Image_uint {{%\d+}} %uint_10 %uint_0
+// CHECK:   [[nu38]] = OpAtomicIAdd
     uint old = 0;
     InterlockedAdd(gRWBuffers[NonUniformResourceIndex(index) * NonUniformResourceIndex(index)][10], 1, old);
 
-// CHECK: [[nu38]] = OpLoad %uint %index
-// CHECK: [[nu39]] = OpAccessChain %_ptr_UniformConstant_type_subpass_image %gSubpassInputs [[nu38]]
-// CHECK: [[nu40]] = OpLoad %type_subpass_image
-// CHECK:            OpImageRead
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu39]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu40]] = OpAccessChain %_ptr_UniformConstant_type_subpass_image %gSubpassInputs [[nu39]]
+// CHECK:   [[nu41]] = OpLoad %type_subpass_image
+// CHECK:              OpImageRead
     float4 v7 = gSubpassInputs[NonUniformResourceIndex(index)].SubpassLoad();
 
-// CHECK: [[nu41]] = OpLoad %uint %index
-// CHECK: [[nu42]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu41]]
-// CHECK: [[nu43]] = OpLoad %type_2d_image
-// CHECK: [[nu44]] = OpSampledImage %type_sampled_image
-// CHECK:            OpImageGather
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu42]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu43]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu42]]
+// CHECK:   [[nu44]] = OpLoad %type_2d_image
+// CHECK:   [[nu45]] = OpSampledImage %type_sampled_image
+// CHECK:              OpImageGather
     float4 v8 = gTextures[NonUniformResourceIndex(index)].Gather(gSamplers[0], loc, offset);
 
-// CHECK: [[nu45]] = OpLoad %uint %index
-// CHECK: [[nu46]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu45]]
-// CHECK: [[nu47]] = OpLoad %type_2d_image
-// CHECK: [[nu48]] = OpSampledImage %type_sampled_image
-// CHECK:            OpImageQueryLod
-// CHECK: [[nu49]] = OpLoad %uint %index
-// CHECK: [[nu50]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu49]]
-// CHECK: [[nu51]] = OpLoad %type_2d_image
-// CHECK: [[nu52]] = OpSampledImage %type_sampled_image
-// CHECK:            OpImageQueryLod
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu46]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu47]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu46]]
+// CHECK:   [[nu48]] = OpLoad %type_2d_image
+// CHECK:   [[nu49]] = OpSampledImage %type_sampled_image
+// CHECK:              OpImageQueryLod
+// CHECK: [[i:%\d+]] = OpLoad %uint %index
+// CHECK:   [[nu50]] = OpCopyObject %uint [[i]]
+// CHECK:   [[nu51]] = OpAccessChain %_ptr_UniformConstant_type_2d_image %gTextures [[nu50]]
+// CHECK:   [[nu52]] = OpLoad %type_2d_image
+// CHECK:   [[nu53]] = OpSampledImage %type_sampled_image
+// CHECK:              OpImageQueryLod
     float  v9 = gTextures[NonUniformResourceIndex(index)].CalculateLevelOfDetail(gSamplers[0], 0.5);
 
     return v1 + v2 + v3 + v4 + v5 + v6 + v7;

tools/clang/test/CodeGenSPIRV/spirv.debug.opline.intrinsic.hlsl

@@ -81,9 +81,9 @@ void main() {
 
 // CHECK:                     OpLine [[file]] 87 41
 // CHECK-NEXT: [[idx:%\d+]] = OpIAdd %uint
-// CHECK-NEXT:                OpLine [[file]] 87 3
+// CHECK:                     OpLine [[file]] 87 3
 // CHECK-NEXT: [[v4i:%\d+]] = OpAccessChain %_ptr_Function_uint %v4i %int_0
-// CHECK-NEXT:                OpStore [[v4i]] [[idx]]
+// CHECK-NEXT:                OpStore [[v4i]] {{%\d+}}
   v4i.x = NonUniformResourceIndex(v4i.y + v4i.z);
 
 // CHECK:      OpLine [[file]] 93 11
@@ -179,7 +179,7 @@ void main() {
 // CHECK-NEXT: OpExtInst %uint {{%\d+}} FindUMsb
   max(firstbithigh(sqrt(abs(v2f.x * v4f.w)) + v4i.x),
 // CHECK:      OpLine [[file]] 183 7
-// CHECK-NEXT: OpExtInst %float {{%\d+}} Cos %468
+// CHECK-NEXT: OpExtInst %float {{%\d+}} Cos
       cos(v4f.x));
 // CHECK:      OpLine [[file]] 180 3
 // CHECK-NEXT: OpExtInst %float {{%\d+}} FMax