Generate new version for each DX Compiler build (#2200)

Added generating of new version for each DX Compiler build. 

There are 3 kinds of version:
1. **Official build**
Built by using `hctbuild -official`. The version is based on the current DXIL version, latest official release and a number of commits since then. The format is `dxil_major.dxil_minor.release_no.commit_count`. For example a current official version would be something like `1.5.1905.42`. The latest release information is read from `utils\version\latest-release.json`. The `1905` corresponds to `dxil-2019-05-16` release branch and `42` is the number of commits since that release branch was created. For master branch the `commit_count` will be incremented by 10000 to distinguish it from stabilized official release branch builds. So the current official version of master would be someting like `1.5.1905.10042`.

2. **Dev build**
Build by using `hctbuild` with no other version-related option. The format is `dxil_major.dxil_minor.0.commit_count` where commit_count is the number of total commits since the beginning of the project.

3. **Fixed version build**
Build by using `hctbuild -fv`. Enables overriding of the version information. The fixed version is read from `utils\version\version.inc`. Location of the version file can be overriden by `-fvloc` option on `hctbuild`.

In addition to the numbered version the product version string on the binaries will also include branch name and last commit sha - `"1.5.1905.10042 (master, 47e31c8a)"`. This product version string is included in `dxc -?` output.

CMakeLists.txt

@@ -79,7 +79,20 @@ if ( LLVM_USE_FOLDERS )
   set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 endif()
 
-option(HLSL_ENABLE_FIXED_VER "Sets up fixed version information." OFF) # HLSL Change
+option(HLSL_EMBED_VERSION "If set, embeds version information in the binaries." ON) # HLSL Change
+# HLSL Change - Directory with version.inc file used for the build (generated or copied from HLSL_FIXED_VERSION_LOCATION)
+set(HLSL_VERSION_LOCATION ${CMAKE_BINARY_DIR}\\utils\\version) 
+option(HLSL_OFFICIAL_BUILD "If set, embeds official version in the binaries (based on the latest release version)" OFF) # HLSL Change
+option(HLSL_ENABLE_FIXED_VER "Reads version information from a file." OFF) # HLSL Change
+
+if (HLSL_ENABLE_FIXED_VER)
+  set(HLSL_EMBED_VERSION ON)
+  if (NOT DEFINED HLSL_FIXED_VERSION_LOCATION)
+    # HLSL Change - Directory with fixed version.inc file
+    set(HLSL_FIXED_VERSION_LOCATION  ${CMAKE_SOURCE_DIR}\\utils\\version) 
+  endif()
+endif()
+
 option(HLSL_ENABLE_ANALYZE "Enables compiler analysis during compilation." OFF) # HLSL Change
 option(HLSL_OPTIONAL_PROJS_IN_DEFAULT "Include optional projects in default build target." OFF) # HLSL Change
 
@@ -617,6 +630,10 @@ add_subdirectory(include/dxc/Tracing) # HLSL Change
 
 add_subdirectory(lib)
 
+if (HLSL_EMBED_VERSION) 
+  add_subdirectory(utils/version)
+endif()
+
 if( LLVM_INCLUDE_UTILS )
   # add_subdirectory(utils/FileCheck) # HLSL Change
   # add_subdirectory(utils/FileUpdate) # HLSL Change

cmake/modules/AddLLVM.cmake

@@ -954,20 +954,15 @@ endfunction()
 
 # HLSL Change Starts
 function(hlsl_update_product_ver RC_INTERNAL_NAME)
-  if (HLSL_ENABLE_FIXED_VER)
+  if(HLSL_EMBED_VERSION)
     set_property(SOURCE ${windows_resource_file}
                  PROPERTY COMPILE_DEFINITIONS
-                 "RC_COMPANY_NAME=\"Microsoft(r) Corporation\""
-                 "RC_VERSION_FIELD_1=0"
-                 "RC_VERSION_FIELD_2=2019"
-                 "RC_VERSION_FIELD_3=05"
-                 "RC_VERSION_FIELD_4=00"
-                 "RC_FILE_VERSION=\"0.2019.05.00\""
-                 "RC_FILE_DESCRIPTION=\"DirectX Compiler - Out Of Band\""
-                 "RC_INTERNAL_NAME=\"${RC_INTERNAL_NAME}\""
-                 "RC_COPYRIGHT=\"(c) Microsoft Corporation. All rights reserved.\""
-                 "RC_PRODUCT_NAME=\"Microsoft(r) DirectX for Windows(r) - Out Of Band\""
-                 "RC_PRODUCT_VERSION=\"0.2019.05.00\"")
-  endif (HLSL_ENABLE_FIXED_VER)
+                 "INCLUDE_HLSL_VERSION_FILE=1"
+                 "RC_INTERNAL_NAME=\"${RC_INTERNAL_NAME}\"")
+    set_property(SOURCE ${windows_resource_file}
+                 PROPERTY COMPILE_OPTIONS
+                 "/i" "${HLSL_VERSION_LOCATION}")
+    add_dependencies(${RC_INTERNAL_NAME} hlsl_version_autogen)
+  endif(HLSL_EMBED_VERSION)
 endfunction(hlsl_update_product_ver)
 # HLSL Change Ends

resources/windows_version_resource.rc

@@ -1,3 +1,4 @@
+
 // Microsoft Visual C++ resource script for embedding version information.
 // The format is described at:
 //   http://msdn.microsoft.com/en-gb/library/windows/desktop/aa380599(v=vs.85).aspx
@@ -51,6 +52,9 @@
 #define RC_PRODUCT_VERSION ""
 #endif
 
+#ifdef INCLUDE_HLSL_VERSION_FILE
+#include "version.inc"
+#endif
 
 1 VERSIONINFO
 FILEVERSION RC_VERSION_FIELD_1,RC_VERSION_FIELD_2,RC_VERSION_FIELD_3,RC_VERSION_FIELD_4

tools/clang/tools/dxc/dxclib/dxc.cpp

@@ -992,7 +992,7 @@ bool GetDLLFileVersionInfo(const char *dllPath, unsigned int *version) {
   DWORD dwVerHnd = 0;
   DWORD size = GetFileVersionInfoSize(dllPath, &dwVerHnd);
   if (size == 0) return false;
-  std::unique_ptr<int[]> VfInfo(new int[size]);
+  std::unique_ptr<BYTE[]> VfInfo(new BYTE[size]);
   if (GetFileVersionInfo(dllPath, NULL, size, VfInfo.get())) {
       LPVOID versionInfo;
       UINT size;
@@ -1011,6 +1011,27 @@ bool GetDLLFileVersionInfo(const char *dllPath, unsigned int *version) {
   return false;
 }
 
+bool GetDLLProductVersionInfo(const char *dllPath, std::string &productVersion) {
+  // This function is used to get product version information from the DLL file.
+  // This information in is not available through a Unix interface.
+#ifdef _WIN32
+  DWORD dwVerHnd = 0;
+  DWORD size = GetFileVersionInfoSize(dllPath, &dwVerHnd);
+  if (size == 0) return false;
+  std::unique_ptr<BYTE[]> VfInfo(new BYTE[size]);
+  if (GetFileVersionInfo(dllPath, NULL, size, VfInfo.get())) {
+    LPVOID pvProductVersion = NULL;
+    unsigned int iProductVersionLen = 0;
+    // 040904b0 == code page US English, Unicode
+    if (VerQueryValue(VfInfo.get(), "\\StringFileInfo\\040904b0\\ProductVersion", &pvProductVersion, &iProductVersionLen)) {
+      productVersion = (LPCSTR)pvProductVersion;
+      return true;
+    }
+  }
+#endif // _WIN32
+  return false;
+}
+
 // Collects compiler/validator version info
 void DxcContext::GetCompilerVersionInfo(llvm::raw_string_ostream &OS) {
   if (m_dxcSupport.IsEnabled()) {
@@ -1025,7 +1046,7 @@ void DxcContext::GetCompilerVersionInfo(llvm::raw_string_ostream &OS) {
 #endif // SUPPORT_QUERY_GIT_COMMIT_INFO
 
     const char *compilerName =
-        m_Opts.ExternalFn.empty() ? "dxcompiler.dll" : m_Opts.ExternalFn.data();
+      m_Opts.ExternalFn.empty() ? "dxcompiler.dll" : m_Opts.ExternalFn.data();
 
     if (SUCCEEDED(CreateInstance(CLSID_DxcCompiler, &VerInfo))) {
       VerInfo->GetVersion(&compilerMajor, &compilerMinor);
@@ -1043,24 +1064,27 @@ void DxcContext::GetCompilerVersionInfo(llvm::raw_string_ostream &OS) {
 #ifdef _WIN32
     unsigned int version[4];
     if (GetDLLFileVersionInfo(compilerName, version)) {
-      // unofficial version always have file version 3.7.0.0
-      if (version[0] == 3 && version[1] == 7 && version[2] == 0 &&
-          version[3] == 0) {
-#endif // _WIN32
+      // back-compat - old dev buidls had version 3.7.0.0
+      if (version[0] == 3 && version[1] == 7 && version[2] == 0 && version[3] == 0) {
+#endif
         OS << "(dev"
 #ifdef SUPPORT_QUERY_GIT_COMMIT_INFO
-           << ";" << commitCount << "-"
-           << (commitHash.m_pData ? commitHash.m_pData : "<unknown-git-hash>")
-#endif // SUPPORT_QUERY_GIT_COMMIT_INFO
-           << ")";
+          << ";" << commitCount << "-"
+          << (commitHash.m_pData ? commitHash.m_pData : "<unknown-git-hash>")
+#endif // SUPPORT_QUERY_GIT_COMMIT_I#else 
+          << ")";
 #ifdef _WIN32
-      } else {
-        OS << "(" << version[0] << "." << version[1] << "." << version[2] << "."
-           << version[3] << ")";
+      }
+      else {
+        std::string productVersion;
+        if (GetDLLProductVersionInfo(compilerName, productVersion)) {
+          OS << " - " << productVersion;
+        }
       }
     }
-#endif // _WIN32
+#endif
   }
+
   // Print validator if exists
   DxcDllSupport DxilSupport;
   DxilSupport.InitializeForDll(L"dxil.dll", "DxcCreateInstance");

utils/hct/hctbuild.cmd

@@ -65,11 +65,22 @@ if "%1"=="-analyze" (
   set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_ENABLE_ANALYZE:BOOL=ON
   shift /1
 )
+if "%1"=="-official" (
+  echo Will generate official version for build
+  set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_OFFICIAL_BUILD:BOOL=ON
+  shift /1
+)
 if "%1"=="-fv" (
   echo Fixed version flag set for build.
   set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_ENABLE_FIXED_VER:BOOL=ON
   shift /1
 )
+if "%1"=="-fvloc" (
+  echo Fixed version flag set for build, version file location: %2
+  set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_ENABLE_FIXED_VER:BOOL=ON -DHLSL_FIXED_VERSION_LOCATION:STRING=%2
+  shift /1
+  shift /1
+)
 if "%1"=="-cv" (
   echo Set the CLANG_VENDOR value.
   set CMAKE_OPTS=%CMAKE_OPTS% -DCLANG_VENDOR:STRING=%2
@@ -251,18 +262,19 @@ exit /b 0
 echo Builds HLSL solutions and the product and test binaries for the current
 echo flavor and architecture.
 echo.
-echo hctbuild [-s or -b] [-alldef] [-analyze] [-fv] [-rel] [-arm or -arm64 or -x86 or -x64] [-Release] [-Debug] [-vs2015] [-ninja] [-tblgen path] [-dont-speak] [-parallel]
+echo hctbuild [-s or -b] [-alldef] [-analyze] [-official] [-fv] [-fvloc <path>] [-rel] [-arm or -arm64 or -x86 or -x64] [-Release] [-Debug] [-vs2015] [-ninja] [-tblgen path] [-dont-speak] [-parallel]
 echo.
 echo   -s   creates the projects only, without building
 echo   -b   builds the existing project
 echo.
-echo   -alldef  adds optional projects to the default build
-echo   -analyze adds /analyze option
-echo   -fv      fixes the resource version for release
-echo.
-echo   -rel     builds release rather than debug
-echo   -dont-speak  disables audible build confirmation
-echo   -parallel    enables parallel build
+echo   -alldef        adds optional projects to the default build
+echo   -analyze       adds /analyze option
+echo   -official      will generate official version for build
+echo   -fv            fixes the resource version for release (utils\version\version.inc)
+echo   -fvloc <path>  directory with the version.inc file
+echo   -rel           builds release rather than debug
+echo   -dont-speak    disables audible build confirmation
+echo   -parallel      enables parallel build
 echo.
 echo current BUILD_ARCH=%BUILD_ARCH%.  Override with:
 echo   -x86 targets an x86 build (aka. Win32)

utils/version/CMakeLists.txt

@@ -0,0 +1,34 @@
+if(HLSL_EMBED_VERSION)
+    string(REPLACE "/" "\\" HLSL_VERSION_LOCATION_mspath ${HLSL_VERSION_LOCATION})
+
+    set(HLSL_VERSION_FILE ${HLSL_VERSION_LOCATION_mspath}\\version.inc)
+    if(HLSL_ENABLE_FIXED_VER)
+        string(REPLACE "/" "\\" HLSL_FIXED_VERSION_LOCATION_mspath ${HLSL_FIXED_VERSION_LOCATION})
+        message("Using fixed version file ${HLSL_FIXED_VERSION_LOCATION_mspath}\\version.inc")
+        add_custom_target(hlsl_version_autogen
+            COMMAND echo "Using fixed version file ${HLSL_FIXED_VERSION_LOCATION_mspath}\\version.inc"
+            COMMAND copy ${HLSL_FIXED_VERSION_LOCATION_mspath}\\version.inc ${HLSL_VERSION_LOCATION_mspath}
+            )
+    else(HLSL_ENABLE_FIXED_VER)
+        if (HLSL_OFFICIAL_BUILD)
+            message("Will generate official build version based on the latest release fork sha and current commit count")
+            set(GEN_VERSION_OFFICIAL_OPTION "--official")
+        else (HLSL_OFFICIAL_BUILD)
+            message("Will generate dev build version based on current commit count")
+            set(GEN_VERSION_OFFICIAL_OPTION "")
+        endif(HLSL_OFFICIAL_BUILD)
+
+        add_custom_target(hlsl_version_autogen
+            COMMAND echo "Generating version"
+            COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/gen_version.py ${GEN_VERSION_OFFICIAL_OPTION} > ${HLSL_VERSION_FILE}
+        )
+        set_property(SOURCE ${HLSL_VERSION_FILE} 
+            PROPERTY GENERATED True
+        )
+    endif(HLSL_ENABLE_FIXED_VER)
+    
+    set_target_properties(hlsl_version_autogen PROPERTIES
+        FOLDER version
+    )
+
+endif(HLSL_EMBED_VERSION)

utils/version/gen_version.py

@@ -0,0 +1,133 @@
+# Added generating of new version for each DX Compiler build. 
+
+# There are 3 kinds of version:
+# 1. **Official build**
+# Built by using `hctbuild -official`. The version is based on the current DXIL version, latest official 
+# release and a number of commits since then. The format is `dxil_major.dxil_minor.release_no.commit_count`.
+# For example a current official version would be something like `1.5.1905.42`. The latest release 
+# information is read from `utils\version\latest-release.json`. The `1905` corresponds to `dxil-2019-05-16`
+# release branch and `42` is the number of commits since that release branch was created. For master branch 
+# the `commit_count` will be incremented by 10000 to distinguish it from stabilized official release branch 
+# builds. So the current official version of master would be someting like `1.5.1905.10042`.
+
+# 2. **Dev build**
+# Build by using `hctbuild` with no other version-related option. 
+# The format is `dxil_major.dxil_minor.0.commit_count` where commit_count is the number of total commits 
+# since the beginning of the project.
+
+# 3. **Fixed version build**
+# Build by using `hctbuild -fv`. Enables overriding of the version information. The fixed version is 
+# read from `utils\version\version.inc`. Location of the version file can be overriden by `-fvloc` option
+# on `hctbuild`.
+
+# In addition to the numbered version the product version string on the binaries will also include branch
+# name and last commit sha - `"1.5.1905.10042 (master, 47e31c8a)"`. This product version string is included 
+# in `dxc -?` output.
+
+import argparse
+import json
+import os
+import re
+import subprocess
+
+def get_output_of(cmd):
+    enlistment_root=os.path.dirname(os.path.abspath( __file__ ))
+    output = subprocess.check_output(cmd, cwd=enlistment_root)
+    return output.decode('ASCII').strip()
+
+def get_last_commit_sha():
+    return get_output_of([ "git", "describe", "--always", "--dirty" ])
+
+def get_current_branch():
+    return get_output_of([ "git", "rev-parse", "--abbrev-ref", "HEAD" ])
+
+def get_commit_count(sha):
+    return get_output_of([ "git", "rev-list", "--count", sha ])
+
+def read_latest_release_info():
+    latest_release_file = os.path.join(os.path.dirname(os.path.abspath( __file__)), "latest-release.json")
+    with open(latest_release_file, 'r') as f:
+        return json.load(f)
+
+class VersionGen():
+    def __init__(self, options):
+        self.options = options
+        self.latest_release_info = read_latest_release_info()
+        self.current_branch = get_current_branch()
+        self.rc_version_field_4_cache = None
+
+    def rc_version_field_1(self):
+        return self.latest_release_info["version"]["major"]
+
+    def rc_version_field_2(self):
+        return self.latest_release_info["version"]["minor"]
+
+    def rc_version_field_3(self):
+        return self.latest_release_info["version"]["rev"] if self.options.official else "0"
+
+    def rc_version_field_4(self):
+        if self.rc_version_field_4_cache is None:
+            base_commit_count = 0
+            if self.options.official:
+                base_commit_count = int(get_commit_count(self.latest_release_info["sha"]))
+            current_commit_count = int(get_commit_count("HEAD"))
+            distance_from_base = current_commit_count - base_commit_count
+            if (self.current_branch is "master"):
+                distance_from_base += 10000
+            self.rc_version_field_4_cache = str(distance_from_base)
+        return self.rc_version_field_4_cache
+
+    def quoted_version_str(self):
+        return '"{}.{}.{}.{}"'.format(
+            self.rc_version_field_1(),
+            self.rc_version_field_2(),
+            self.rc_version_field_3(),
+            self.rc_version_field_4())
+
+    def product_version_str(self):
+        if (self.options.no_commit_sha):
+            return self.quoted_version_str()
+        else:
+            return '"{}.{}.{}.{} ({}, {})"'.format(
+                self.rc_version_field_1(),
+                self.rc_version_field_2(),
+                self.rc_version_field_3(),
+                self.rc_version_field_4(),
+                self.current_branch,
+                get_last_commit_sha()
+                )
+
+    def print_define(self, name, value):
+        print('#ifdef {}'.format(name))
+        print('#undef {}'.format(name))
+        print('#endif')
+        print('#define {} {}'.format(name, value))
+        print()
+
+    def print_version(self):
+        print('#pragma once')
+        print()
+        self.print_define('RC_COMPANY_NAME',      '"Microsoft(r) Corporation"')
+        self.print_define('RC_VERSION_FIELD_1',   self.rc_version_field_1())
+        self.print_define('RC_VERSION_FIELD_2',   self.rc_version_field_2())
+        self.print_define('RC_VERSION_FIELD_3',   self.rc_version_field_3() if self.options.official else "0")
+        self.print_define('RC_VERSION_FIELD_4',   self.rc_version_field_4())
+        self.print_define('RC_FILE_VERSION',      self.quoted_version_str())
+        self.print_define('RC_FILE_DESCRIPTION', '"DirectX Compiler - Out Of Band"')
+        self.print_define('RC_COPYRIGHT',        '"(c) Microsoft Corporation. All rights reserved."')
+        self.print_define('RC_PRODUCT_NAME',     '"Microsoft(r) DirectX for Windows(r) - Out Of Band"')
+        self.print_define('RC_PRODUCT_VERSION',   self.product_version_str())
+
+
+def main():
+    p = argparse.ArgumentParser("gen_version")
+    p.add_argument("--no-commit-sha", action='store_true')
+    p.add_argument("--official", action="store_true")
+    args = p.parse_args()
+
+    VersionGen(args).print_version() 
+
+
+if __name__ == '__main__':
+    main()
+

utils/version/latest-release.json

@@ -0,0 +1,8 @@
+{
+    "version" : {
+        "major" : "1",
+        "minor" : "5",
+        "rev"   : "1905"
+    },
+    "sha" : "fdd4e3b0e6f1ba7a22572c02f95450288711b8ce"
+}

utils/version/version.inc

@@ -0,0 +1,52 @@
+#pragma once
+
+#ifdef RC_COMPANY_NAME
+#undef RC_COMPANY_NAME
+#endif
+#define RC_COMPANY_NAME "Microsoft(r) Corporation"
+
+#ifdef RC_VERSION_FIELD_1
+#undef RC_VERSION_FIELD_1
+#endif
+#define RC_VERSION_FIELD_1 1
+
+#ifdef RC_VERSION_FIELD_2
+#undef RC_VERSION_FIELD_2
+#endif
+#define RC_VERSION_FIELD_2 4
+
+#ifdef RC_VERSION_FIELD_3
+#undef RC_VERSION_FIELD_3
+#endif
+#define RC_VERSION_FIELD_3 1905
+
+#ifdef RC_VERSION_FIELD_4
+#undef RC_VERSION_FIELD_4
+#endif
+#define RC_VERSION_FIELD_4 1
+
+#ifdef RC_FILE_VERSION
+#undef RC_FILE_VERSION
+#endif
+#define RC_FILE_VERSION "1.4.1905.1"
+
+#ifdef RC_FILE_DESCRIPTION
+#undef RC_FILE_DESCRIPTION
+#endif
+#define RC_FILE_DESCRIPTION "DirectX Compiler - Out Of Band"
+
+#ifdef RC_COPYRIGHT
+#undef RC_COPYRIGHT
+#endif
+#define RC_COPYRIGHT "(c) Microsoft Corporation. All rights reserved."
+
+#ifdef RC_PRODUCT_NAME
+#undef RC_PRODUCT_NAME
+#endif
+#define RC_PRODUCT_NAME "Microsoft(r) DirectX for Windows(r) - Out Of Band"
+
+#ifdef RC_PRODUCT_VERSION
+#undef RC_PRODUCT_VERSION
+#endif
+#define RC_PRODUCT_VERSION "1.4.1905.1"
+