Merged PR 87: Fix matrix orientation and unorm/snorm detection in codegen.

tools/clang/include/clang/AST/HlslTypes.h

@@ -364,6 +364,9 @@ ConvertHLSLVecMatTypeToExtVectorType(const clang::ASTContext &,
 bool IsHLSLVecMatType(clang::QualType);
 bool IsHLSLVecType(clang::QualType type);
 bool IsHLSLMatType(clang::QualType type);
+clang::QualType GetElementTypeOrType(clang::QualType type);
+bool HasHLSLMatOrientation(clang::QualType type, bool *pIsRowMajor);
+bool HasHLSLUNormSNorm(clang::QualType type, bool *pIsSNorm);
 bool IsHLSLInputPatchType(clang::QualType type);
 bool IsHLSLOutputPatchType(clang::QualType type);
 bool IsHLSLPointStreamType(clang::QualType type);

tools/clang/lib/AST/HlslTypes.cpp

@@ -124,6 +124,70 @@ bool IsHLSLNumericUserDefinedType(clang::QualType type) {
   return false;
 }
 
+clang::QualType GetElementTypeOrType(clang::QualType type) {
+  if (const RecordType *RT = type->getAs<RecordType>()) {
+    if (const ClassTemplateSpecializationDecl *templateDecl =
+      dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl())) {
+      // TODO: check pointer instead of name
+      if (templateDecl->getName() == "vector") {
+        const TemplateArgumentList &argList = templateDecl->getTemplateArgs();
+        return argList[0].getAsType();
+      }
+      else if (templateDecl->getName() == "matrix") {
+        const TemplateArgumentList &argList = templateDecl->getTemplateArgs();
+        return argList[0].getAsType();
+      }
+    }
+  }
+  return type;
+}
+
+bool HasHLSLMatOrientation(clang::QualType type, bool *pIsRowMajor) {
+  const AttributedType *AT = type->getAs<AttributedType>();
+  while (AT) {
+    AttributedType::Kind kind = AT->getAttrKind();
+    switch (kind) {
+    case AttributedType::attr_hlsl_row_major:
+      if (pIsRowMajor) *pIsRowMajor = true;
+      return true;
+    case AttributedType::attr_hlsl_column_major:
+      if (pIsRowMajor) *pIsRowMajor = false;
+      return true;
+    }
+    AT = AT->getLocallyUnqualifiedSingleStepDesugaredType()->getAs<AttributedType>();
+  }
+  return false;
+}
+
+bool HasHLSLUNormSNorm(clang::QualType type, bool *pIsSNorm) {
+  // snorm/unorm can be on outer vector/matrix as well as element type
+  // in the template form.  Outer-most type attribute wins.
+  // The following drills into attributed type for outer type,
+  // setting *pIsSNorm and returning true if snorm/unorm found.
+  // If not found on outer type, fall back to element type if different,
+  // indicating a vector or matrix, and try again.
+  clang::QualType elementType = GetElementTypeOrType(type);
+  while (true) {
+    const AttributedType *AT = type->getAs<AttributedType>();
+    while (AT) {
+      AttributedType::Kind kind = AT->getAttrKind();
+      switch (kind) {
+      case AttributedType::attr_hlsl_snorm:
+        if (pIsSNorm) *pIsSNorm = true;
+        return true;
+      case AttributedType::attr_hlsl_unorm:
+        if (pIsSNorm) *pIsSNorm = false;
+        return true;
+      }
+      AT = AT->getLocallyUnqualifiedSingleStepDesugaredType()->getAs<AttributedType>();
+    }
+    if (type == elementType)
+      break;
+    type = elementType;
+  }
+  return false;
+}
+
 /// Checks whether the pAttributes indicate a parameter is inout or out; if
 /// inout, pIsIn will be set to true.
 bool IsParamAttributedAsOut(_In_opt_ clang::AttributeList *pAttributes,

tools/clang/lib/CodeGen/CGCall.cpp

@@ -2435,7 +2435,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
         // HLSL Change Begins
         if (hlsl::IsHLSLMatType(RetTy))
           RV = CGM.getHLSLRuntime().EmitHLSLMatrixLoad(*this, ReturnValue,
-                                                       RetTy);
+                                      FnRetTy);  // FnRetTy retains attributed type
         else
           // HLSL Change Ends
           RV = Builder.CreateLoad(ReturnValue);
@@ -2468,7 +2468,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
         // HLSL Change Begins
         if (hlsl::IsHLSLMatType(RetTy))
           RV = CGM.getHLSLRuntime().EmitHLSLMatrixLoad(*this, ReturnValue,
-                                                       RetTy);
+                                      FnRetTy);  // FnRetTy retains attributed type
         else
           // HLSL Change Ends
           RV = Builder.CreateLoad(ReturnValue);

tools/clang/lib/CodeGen/CGHLSLMS.cpp

@@ -781,21 +781,11 @@ void CGMSHLSLRuntime::ConstructFieldAttributedAnnotation(
   QualType EltTy = Ty;
   if (hlsl::IsHLSLMatType(Ty)) {
     DxilMatrixAnnotation Matrix;
-    Matrix.Orientation = bDefaultRowMajor ? MatrixOrientation::RowMajor
-                                          : MatrixOrientation::ColumnMajor;
-    if (const AttributedType *AT = dyn_cast<AttributedType>(Ty)) {
-      switch (AT->getAttrKind()) {
-      case AttributedType::Kind::attr_hlsl_column_major:
-        Matrix.Orientation = MatrixOrientation::ColumnMajor;
-        break;
-      case AttributedType::Kind::attr_hlsl_row_major:
-        Matrix.Orientation = MatrixOrientation::RowMajor;
-        break;
-      default:
-        // Do nothing
-        break;
-      }
-    }
+    bool bRowMajor = bDefaultRowMajor;
+    HasHLSLMatOrientation(Ty, &bRowMajor);
+    Matrix.Orientation = bRowMajor ? MatrixOrientation::RowMajor
+                                   : MatrixOrientation::ColumnMajor;
+
 
     hlsl::GetHLSLMatRowColCount(Ty, Matrix.Rows, Matrix.Cols);
     fieldAnnotation.SetMatrixAnnotation(Matrix);
@@ -812,20 +802,8 @@ void CGMSHLSLRuntime::ConstructFieldAttributedAnnotation(
 
   bool bSNorm = false;
   bool bUNorm = false;
-
-  if (const AttributedType *AT = dyn_cast<AttributedType>(Ty)) {
-    switch (AT->getAttrKind()) {
-    case AttributedType::Kind::attr_hlsl_snorm:
-      bSNorm = true;
-      break;
-    case AttributedType::Kind::attr_hlsl_unorm:
-      bUNorm = true;
-      break;
-    default:
-      // Do nothing
-      break;
-    }
-  }
+  if (HasHLSLUNormSNorm(Ty, &bSNorm) && !bSNorm)
+    bUNorm = true;
 
   if (EltTy->isBuiltinType()) {
     const BuiltinType *BTy = EltTy->getAs<BuiltinType>();
@@ -4714,16 +4692,9 @@ static const HLUnaryOpcode UnaryOperatorKindMap[] = {
 };
 
 static bool IsRowMajorMatrix(QualType Ty, bool bDefaultRowMajor) {
-  if (const AttributedType *AT = Ty->getAs<AttributedType>()) {
-    if (AT->getAttrKind() == AttributedType::attr_hlsl_row_major)
-      return true;
-    else if (AT->getAttrKind() == AttributedType::attr_hlsl_column_major)
-      return false;
-    else
-      return bDefaultRowMajor;
-  } else {
-    return bDefaultRowMajor;
-  }
+  bool bRowMajor = bDefaultRowMajor;
+  HasHLSLMatOrientation(Ty, &bRowMajor);
+  return bRowMajor;
 }
 
 static bool IsUnsigned(QualType Ty) {