Add missing UTF8 for source and output filenames (#2674)

* Add missing UTF8 for source and output filenames

- These need to be converted back to UTF-16 after argument processing for
  a couple APIs and CA2W constructors were missing CP_UTF8 for the input
  encoding from the utf-8 argument strings.

* Add CP_UTF8 to more CW2A and CA2W uses

lib/DxilDia/DxilDiaSession.cpp

@@ -353,7 +353,7 @@ STDMETHODIMP dxil_dia::Session::findInjectedSource(
   /* [in] */ LPCOLESTR srcFile,
   /* [out] */ IDiaEnumInjectedSources **ppResult) {
   if (Contents() != nullptr) {
-    CW2A pUtf8FileName(srcFile);
+    CW2A pUtf8FileName(srcFile, CP_UTF8);
     DxcThreadMalloc TM(m_pMalloc);
     IDiaTable *pTable;
     IFT(Table::Create(this, Table::Kind::InjectedSource, &pTable));

lib/DxilDia/DxilDiaSymbolManager.cpp

@@ -286,7 +286,7 @@ public:
         IMalloc *pMalloc = pSession->GetMallocNoRef();
         IFR(FunctionSymbol::Create(pMalloc, pSession, m_ID, m_Node, m_TypeID, m_Node->getType(), ppRet));
         (*ppRet)->SetLexicalParent(m_ParentID);
-        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str()));
+        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8));
         return S_OK;
     }
 
@@ -378,7 +378,7 @@ public:
         IMalloc *pMalloc = pSession->GetMallocNoRef();
         IFR(TypedefTypeSymbol::Create(pMalloc, pSession, m_ParentID, m_ID, m_Node, m_BaseTypeID, ppRet));
         (*ppRet)->SetLexicalParent(m_ParentID);
-        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str()));
+        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8));
         return S_OK;
     }
 
@@ -413,7 +413,7 @@ public:
         IMalloc *pMalloc = pSession->GetMallocNoRef();
         IFR(VectorTypeSymbol::Create(pMalloc, pSession, m_ParentID, m_ID, m_Node, m_ElemTyID, m_NumElts, ppRet));
         (*ppRet)->SetLexicalParent(m_ParentID);
-        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str()));
+        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8));
         return S_OK;
     }
 
@@ -468,7 +468,7 @@ public:
         IMalloc *pMalloc = pSession->GetMallocNoRef();
         IFR(GlobalVariableSymbol::Create(pMalloc, pSession, m_ID, m_GV, m_TypeID, m_Type, ppRet));
         (*ppRet)->SetLexicalParent(m_ParentID);
-        (*ppRet)->SetName(CA2W(m_GV->getName().str().c_str()));
+        (*ppRet)->SetName(CA2W(m_GV->getName().str().c_str(), CP_UTF8));
         (*ppRet)->SetIsHLSLData(true);
         return S_OK;
     }
@@ -538,7 +538,7 @@ public:
         IMalloc *pMalloc = pSession->GetMallocNoRef();
         IFR(LocalVariableSymbol::Create(pMalloc, pSession, m_ID, m_Node, m_TypeID, m_Type, m_VI->GetOffsetInUDT(), m_VI->GetDxilRegister(), ppRet));
         (*ppRet)->SetLexicalParent(m_ParentID);
-        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str()));
+        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8));
         (*ppRet)->SetDataKind(m_Node->getTag() == llvm::dwarf::DW_TAG_arg_variable ? DataIsParam : DataIsLocal);
         return S_OK;
     }
@@ -572,7 +572,7 @@ public:
         IMalloc *pMalloc = pSession->GetMallocNoRef();
         IFR(UDTFieldSymbol::Create(pMalloc, pSession, m_ID, m_Node, m_TypeID, m_Type, ppRet));
         (*ppRet)->SetLexicalParent(m_ParentID);
-        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str()));
+        (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8));
         (*ppRet)->SetDataKind(m_Node->isStaticMember() ? DataIsStaticLocal : DataIsMember);
         return S_OK;
     }
@@ -841,7 +841,7 @@ HRESULT dxil_dia::hlsl_symbols::CompilandEnvSymbol::CreateDefines(IMalloc *pMall
   for (llvm::MDNode::op_iterator it = definesNode->op_begin(); it != definesNode->op_end(); ++it) {
     llvm::StringRef strRef = llvm::dyn_cast<llvm::MDString>(*it)->getString();
     std::string str(strRef.begin(), strRef.size());
-    CA2W cv(str.c_str());
+    CA2W cv(str.c_str(), CP_UTF8);
     pBSTR.Append(cv);
     pBSTR.Append(L"\0", 1);
   }
@@ -904,7 +904,7 @@ STDMETHODIMP dxil_dia::hlsl_symbols::TypeSymbol::get_name(
     DXASSERT(!this->HasName(), "Setting type name multiple times.");
     std::string Name;
     IFR(m_lazySymbolName(m_pSession, &Name));
-    this->SetName(CA2W(Name.c_str()));
+    this->SetName(CA2W(Name.c_str(), CP_UTF8));
     m_lazySymbolName = nullptr;
   }
   return Symbol::get_name(pRetVal);
@@ -1349,7 +1349,7 @@ HRESULT dxil_dia::hlsl_symbols::SymbolManagerInit::CreateSubroutineType(DWORD dw
           if (!name) {
             OS << "???";
           } else {
-            OS << CW2A((BSTR)name);
+            OS << CW2A((BSTR)name, CP_UTF8);
           }
         }
         if (first) {
@@ -1416,7 +1416,7 @@ HRESULT dxil_dia::hlsl_symbols::SymbolManagerInit::CreateCompositeType(DWORD dwP
       if (!name) {
         OS << "???";
       } else {
-        OS << CW2A((BSTR)name);
+        OS << CW2A((BSTR)name, CP_UTF8);
       }
 
       OS << "[";
@@ -1613,7 +1613,7 @@ HRESULT dxil_dia::hlsl_symbols::SymbolManagerInit::HandleDerivedType(DWORD dwPar
     if (!name) {
       OS << "???";
     } else {
-      OS << CW2A((BSTR)name);
+      OS << CW2A((BSTR)name, CP_UTF8);
     }
     OS << Qualifier;
     return S_OK;

tools/clang/lib/Sema/SemaHLSL.cpp

@@ -3886,7 +3886,7 @@ public:
       const HLSL_INTRINSIC *pIntrinsic = nullptr;
       const HLSL_INTRINSIC *pPrior = nullptr;
       UINT64 lookupCookie = 0;
-      CA2W wideTypeName(typeName);
+      CA2W wideTypeName(typeName, CP_UTF8);
       HRESULT found = table->LookupIntrinsic(wideTypeName, L"*", &pIntrinsic, &lookupCookie);
       while (pIntrinsic != nullptr && SUCCEEDED(found)) {
         if (!AreIntrinsicTemplatesEquivalent(pIntrinsic, pPrior)) {

tools/clang/tools/d3dcomp/d3dcomp.cpp

@@ -52,8 +52,8 @@ HRESULT CompileFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName,
   }
 
   try {
-    CA2W pEntrypointW(pEntrypoint);
-    CA2W pTargetProfileW(pTarget);
+    CA2W pEntrypointW(pEntrypoint, CP_UTF8);
+    CA2W pTargetProfileW(pTarget, CP_UTF8);
     std::vector<std::wstring> defineValues;
     std::vector<DxcDefine> defines;
     if (pDefines) {
@@ -61,9 +61,9 @@ HRESULT CompileFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName,
 
       // Convert to UTF-16.
       while (pCursor->Name) {
-        defineValues.push_back(std::wstring(CA2W(pCursor->Name)));
+        defineValues.push_back(std::wstring(CA2W(pCursor->Name, CP_UTF8)));
         if (pCursor->Definition)
-          defineValues.push_back(std::wstring(CA2W(pCursor->Definition)));
+          defineValues.push_back(std::wstring(CA2W(pCursor->Definition, CP_UTF8)));
         else
           defineValues.push_back(std::wstring());
         ++pCursor;
@@ -154,7 +154,7 @@ HRESULT WINAPI BridgeD3DCompile(LPCVOID pSrcData, SIZE_T SrcDataSize,
   }
 
   try {
-    CA2W pFileName(pSourceName);
+    CA2W pFileName(pSourceName, CP_UTF8);
     return CompileFromBlob(source, pFileName, pDefines, includeHandler, pEntrypoint,
                            pTarget, Flags1, Flags2, ppCode, ppErrorMsgs);
   } catch (const std::bad_alloc &) {
@@ -289,9 +289,9 @@ HRESULT PreprocessFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName,
 
       // Convert to UTF-16.
       while (pCursor->Name) {
-        defineValues.push_back(std::wstring(CA2W(pCursor->Name)));
+        defineValues.push_back(std::wstring(CA2W(pCursor->Name, CP_UTF8)));
         if (pCursor->Definition)
-          defineValues.push_back(std::wstring(CA2W(pCursor->Definition)));
+          defineValues.push_back(std::wstring(CA2W(pCursor->Definition, CP_UTF8)));
         else
           defineValues.push_back(std::wstring());
         ++pCursor;
@@ -357,7 +357,7 @@ HRESULT WINAPI BridgeD3DPreprocess(_In_reads_bytes_(SrcDataSize) LPCVOID pSrcDat
   }
 
   try {
-    CA2W pFileName(pSourceName);
+    CA2W pFileName(pSourceName, CP_UTF8);
     return PreprocessFromBlob(source, pFileName, pDefines, includeHandler,
                               ppCodeText, ppErrorMsgs);
   } catch (const std::bad_alloc &) {

tools/clang/tools/dxa/dxa.cpp

@@ -207,7 +207,7 @@ bool DxaContext::ExtractFile(const char *pName) {
     return printedAny;
   }
 
-  CA2W WideName(pName);
+  CA2W WideName(pName, CP_UTF8);
   for (;;) {
     CComPtr<IUnknown> pInjectedSourceUnk;
     ULONG fetched;

tools/clang/tools/dxcompiler/dxcfilesystem.cpp

@@ -775,7 +775,7 @@ public:
   }
 
   virtual int Open(const char *lpFileName, int flags, mode_t mode) throw() override {
-    HANDLE H = CreateFileW(CA2W(lpFileName), GENERIC_READ | GENERIC_WRITE,
+    HANDLE H = CreateFileW(CA2W(lpFileName, CP_UTF8), GENERIC_READ | GENERIC_WRITE,
                            FILE_SHARE_READ | FILE_SHARE_WRITE,
                            OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL);
     if (H == INVALID_HANDLE_VALUE)
@@ -788,7 +788,7 @@ public:
 
   // fake my way toward as linux-y a file_status as I can get
   virtual int Stat(const char *lpFileName, struct stat *Status) throw() override {
-    CA2W fileName_utf16(lpFileName);
+    CA2W fileName_utf16(lpFileName, CP_UTF8);
 
     DWORD attr = GetFileAttributesW(fileName_utf16);
     if (attr == INVALID_FILE_ATTRIBUTES)

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -509,14 +509,15 @@ public:
 
       // Formerly API values.
       const char *pUtf8SourceName = opts.InputFile.empty() ? "hlsl.hlsl" : opts.InputFile.data();
-      CA2W pUtf16SourceName(pUtf8SourceName);
+      CA2W pUtf16SourceName(pUtf8SourceName, CP_UTF8);
       const char *pUtf8EntryPoint = opts.EntryPoint.empty() ? "main" : opts.EntryPoint.data();
       const char *pUtf8OutputName = isPreprocessing
                                     ? opts.Preprocess.data()
                                     : opts.OutputObject.empty()
                                       ? "" : opts.OutputObject.data();
       CA2W pUtf16OutputName(isPreprocessing ?
-                            opts.Preprocess.data() : pUtf8OutputName);
+                              opts.Preprocess.data() : pUtf8OutputName,
+                            CP_UTF8);
       LPCWSTR pObjectName = (!isPreprocessing && opts.OutputObject.empty()) ?
                             nullptr : pUtf16OutputName.m_psz;
       IFT(primaryOutput.SetName(pObjectName));