Browse Source

Bring in code coverage tooling (#4759)

This cherry-picks in a bunch of upstream LLVM changes and has a few small modifications for DXC.

* [CMake] Add LLVM_BUILD_INSTRUMENTED option to enable building with
-fprofile-instr-generate

This is the first step in supporting PGO data generation via CMake.
I've marked the option as advanced and experimental until it is fleshed
out further.

llvm-svn: 255298

* Add support for collating profiles for use with code coverage

Differential Revision: http://reviews.llvm.org/D20993

llvm-svn: 272599

Patch by: Vedant Kumar <[email protected]>

* [CMake] Add targets for generating coverage reports

This is a pretty small bit of CMake goop to generate code coverage
reports. I always forget the right script invocation and end up
fumbling around too much.

Wouldn't it be great to have targets that "Just Work"?

Well, I thought so.

At present this only really works correctly for LLVM, but I'll extend
it in subsequent patches to work for subprojects.

Reviewed By: phosek

Differential Revision: https://reviews.llvm.org/D109019

* Update prepare-code-coverage.py from llvm/main

Theres's a swath of changes that went in here. Cherry-picking them all
indivudially is not worth it. Contained commits:

8773822c578a [Zequan Wu] [Utils] Add -compilation-dir flag to
prepare-code-coverage-artifact.py
577d59bc679d [Nico Weber] svn propset svn:executable on
utils/prepare-code-coverage-artifact.py
9a10eac663dd [Chris Bieneman] [Coverage] Apply filtered paths to summary
16776315460e [Vedant Kumar] [utils] coverage: Add help text about the
--restrict flag (NFC)
f4df0edf3ea3 [Vedant Kumar] [utils] Add a '--unified-report' option to
the code coverage prep script
7466987cb7ba [Vedant Kumar] [utils] Use print_function in the code
coverage prep script, NFC.
273b6dc3e2f7 [Vedant Kumar] [utils] Add an '--only-merge' option to the
code coverage prep script
547ebad0b984 [Vedant Kumar] [utils] Teach the code coverage prep script
about --restrict
05ee94f1b573 [Vedant Kumar] [utils] Generate html reports with the code
coverage utility script
d9aed82cf7e2 [Vedant Kumar] Add support for collating profiles for use
with code coverage

* HLSL-specific change to coverage reporting

This works around DXC not having a cmake target to run tests by
exposing a new variable which can configure which target gets run to
produce profiles. A similar change will be useful upstream, so I'll add
a change there too.
Chris B 2 years ago
parent
commit
3962ff2f80

+ 17 - 0
CMakeLists.txt

@@ -408,6 +408,21 @@ if(CMAKE_CROSSCOMPILING OR (LLVM_OPTIMIZED_TABLEGEN AND LLVM_ENABLE_ASSERTIONS))
   set(LLVM_USE_HOST_TOOLS ON)
 endif()
 
+if (LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE)
+  if(NOT LLVM_PROFILE_MERGE_POOL_SIZE)
+    # A pool size of 1-2 is probably sufficient on a SSD. 3-4 should be fine
+    # for spining disks. Anything higher may only help on slower mediums.
+    set(LLVM_PROFILE_MERGE_POOL_SIZE "4")
+  endif()
+  if(NOT LLVM_PROFILE_FILE_PATTERN)
+    if(NOT LLVM_PROFILE_DATA_DIR)
+      set(LLVM_PROFILE_FILE_PATTERN "%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw")
+    else()
+      file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN)
+    endif()
+  endif()
+endif()
+
 # All options referred to from HandleLLVMOptions have to be specified
 # BEFORE this include, otherwise options will not be correctly set on
 # first cmake run
@@ -768,6 +783,8 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
   endif()
 endif()
 
+include(CoverageReport)
+
 # Disable regeneration target for official builds as they always fun a full clean build.
 # This should eliminate the race-condition issue with "Cannot restore timestamp".
 if (HLSL_OFFICIAL_BUILD)

+ 73 - 0
cmake/modules/CoverageReport.cmake

@@ -0,0 +1,73 @@
+# if coverage reports are not enabled, skip all of this
+if(NOT LLVM_BUILD_INSTRUMENTED_COVERAGE)
+  return()
+endif()
+
+file(TO_NATIVE_PATH
+     "${LLVM_SOURCE_DIR}/utils/prepare-code-coverage-artifact.py"
+     PREPARE_CODE_COV_ARTIFACT)
+
+# llvm-cov and llvm-profdata need to match the host compiler. They can either be
+# explicitly provided by the user, or we will look them up based on the install
+# location of the C++ compiler.
+get_filename_component(COMPILER_DIRECTORY ${CMAKE_CXX_COMPILER} DIRECTORY)
+find_program(LLVM_COV "llvm-cov" ${COMPILER_DIRECTORY} NO_DEFAULT_PATH)
+find_program(LLVM_PROFDATA "llvm-profdata" ${COMPILER_DIRECTORY} NO_DEFAULT_PATH)
+
+if(NOT LLVM_COV OR NOT LLVM_PROFDATA)
+  message(WARNING "Could not find code coverage tools, skipping generating targets. You may explicitly specify LLVM_COV and LLVM_PROFDATA to work around this warning.")
+  return()
+endif()
+
+set(LLVM_CODE_COVERAGE_TARGETS "" CACHE STRING "Targets to generate coverage reports against (defaults to all exported targets if empty)")
+mark_as_advanced(LLVM_CODE_COVERAGE_TARGETS)
+
+# HLSL Change Begin - This is probably worth upstreaming...
+set(LLVM_CODE_COVERAGE_TEST_TARGETS "" CACHE STRING "Targets to run to generate coverage profiles.")
+mark_as_advanced(LLVM_CODE_COVERAGE_TEST_TARGETS)
+
+if (LLVM_CODE_COVERAGE_TEST_TARGETS)
+  set(COV_DEPENDS DEPENDS ${LLVM_CODE_COVERAGE_TEST_TARGETS})
+endif()
+# HLSL Change End
+
+if(NOT LLVM_CODE_COVERAGE_TARGETS)
+  # by default run the coverage report across all the exports provided
+  get_property(COV_TARGETS GLOBAL PROPERTY LLVM_EXPORTS)
+endif()
+
+file(TO_NATIVE_PATH
+     "${CMAKE_BINARY_DIR}/report/"
+     REPORT_DIR)
+
+foreach(target ${LLVM_CODE_COVERAGE_TARGETS} ${COV_TARGETS})
+  get_target_property(target_type ${target} TYPE)
+  if("${target_type}" STREQUAL "SHARED_LIBRARY" OR "${target_type}" STREQUAL "EXECUTABLE")
+    list(APPEND coverage_binaries $<TARGET_FILE:${target}>)
+  endif()
+endforeach()
+
+set(LLVM_COVERAGE_SOURCE_DIRS "" CACHE STRING "Source directories to restrict coverage reports to.")
+mark_as_advanced(LLVM_COVERAGE_SOURCE_DIRS)
+
+foreach(dir ${LLVM_COVERAGE_SOURCE_DIRS})
+  list(APPEND restrict_flags -restrict ${dir})
+endforeach()
+
+# Utility target to clear out profile data.
+# This isn't connected to any dependencies because it is a bit finicky to get
+# working exactly how a user might want.
+add_custom_target(clear-profile-data
+                  COMMAND ${CMAKE_COMMAND} -E
+                          remove_directory ${LLVM_PROFILE_DATA_DIR})
+
+# This currently only works for LLVM, but could be expanded to work for all
+# sub-projects. The current limitation is based on not having a good way to
+# automaticall plumb through the targets that we want to run coverage against.
+add_custom_target(generate-coverage-report
+                  COMMAND ${Python3_EXECUTABLE} ${PREPARE_CODE_COV_ARTIFACT}
+                          ${LLVM_PROFDATA} ${LLVM_COV} ${LLVM_PROFILE_DATA_DIR}
+                          ${REPORT_DIR} ${coverage_binaries}
+                          --unified-report ${restrict_flags}
+                  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+                  ${COV_DEPENDS}) # Run tests

+ 16 - 0
cmake/modules/HandleLLVMOptions.cmake

@@ -620,6 +620,22 @@ if (LLVM_ENABLE_LTO)
 endif()
 # HLSL Change End
 
+option(LLVM_BUILD_INSTRUMENTED "Build LLVM and tools with PGO instrumentation (experimental)" Off)
+mark_as_advanced(LLVM_BUILD_INSTRUMENTED)
+append_if(LLVM_BUILD_INSTRUMENTED "-fprofile-instr-generate='${LLVM_PROFILE_FILE_PATTERN}'"
+  CMAKE_CXX_FLAGS
+  CMAKE_C_FLAGS
+  CMAKE_EXE_LINKER_FLAGS
+  CMAKE_SHARED_LINKER_FLAGS)
+
+option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation (experimental)" Off)
+mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE)
+append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate='${LLVM_PROFILE_FILE_PATTERN}' -fcoverage-mapping"
+  CMAKE_CXX_FLAGS
+  CMAKE_C_FLAGS
+  CMAKE_EXE_LINKER_FLAGS
+  CMAKE_SHARED_LINKER_FLAGS)
+
 # Plugin support
 # FIXME: Make this configurable.
 if(WIN32 OR CYGWIN)

+ 117 - 0
utils/prepare-code-coverage-artifact.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+'''Prepare a code coverage artifact.
+
+- Collate raw profiles into one indexed profile.
+- Generate html reports for the given binaries.
+
+Caution: The positional arguments to this script must be specified before any 
+optional arguments, such as --restrict.
+'''
+
+import argparse
+import glob
+import os
+import subprocess
+import sys
+
+def merge_raw_profiles(host_llvm_profdata, profile_data_dir, preserve_profiles):
+    print(':: Merging raw profiles...', end='')
+    sys.stdout.flush()
+    raw_profiles = glob.glob(os.path.join(profile_data_dir, '*.profraw'))
+    manifest_path = os.path.join(profile_data_dir, 'profiles.manifest')
+    profdata_path = os.path.join(profile_data_dir, 'Coverage.profdata')
+    with open(manifest_path, 'w') as manifest:
+        manifest.write('\n'.join(raw_profiles))
+    subprocess.check_call([host_llvm_profdata, 'merge', '-sparse', '-f',
+                           manifest_path, '-o', profdata_path])
+    if not preserve_profiles:
+        for raw_profile in raw_profiles:
+            os.remove(raw_profile)
+    os.remove(manifest_path)
+    print('Done!')
+    return profdata_path
+
+def prepare_html_report(host_llvm_cov, profile, report_dir, binaries,
+                        restricted_dirs, compilation_dir):
+    print(':: Preparing html report for {0}...'.format(binaries), end='')
+    sys.stdout.flush()
+    objects = []
+    for i, binary in enumerate(binaries):
+        if i == 0:
+            objects.append(binary)
+        else:
+            objects.extend(('-object', binary))
+    invocation = [host_llvm_cov, 'show'] + objects + ['-format', 'html',
+                  '-instr-profile', profile, '-o', report_dir,
+                  '-show-line-counts-or-regions', '-Xdemangler', 'c++filt',
+                  '-Xdemangler', '-n'] + restricted_dirs
+    if compilation_dir:
+        invocation += ['-compilation-dir=' + compilation_dir]
+    subprocess.check_call(invocation)
+    with open(os.path.join(report_dir, 'summary.txt'), 'wb') as Summary:
+        subprocess.check_call([host_llvm_cov, 'report'] + objects +
+                               ['-instr-profile', profile] + restricted_dirs,
+                               stdout=Summary)
+    print('Done!')
+
+def prepare_html_reports(host_llvm_cov, profdata_path, report_dir, binaries,
+                         unified_report, restricted_dirs, compilation_dir):
+    if unified_report:
+        prepare_html_report(host_llvm_cov, profdata_path, report_dir, binaries,
+                            restricted_dirs, compilation_dir)
+    else:
+        for binary in binaries:
+            binary_report_dir = os.path.join(report_dir,
+                                             os.path.basename(binary))
+            prepare_html_report(host_llvm_cov, profdata_path, binary_report_dir,
+                                [binary], restricted_dirs, compilation_dir)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('host_llvm_profdata', help='Path to llvm-profdata')
+    parser.add_argument('host_llvm_cov', help='Path to llvm-cov')
+    parser.add_argument('profile_data_dir',
+                       help='Path to the directory containing the raw profiles')
+    parser.add_argument('report_dir',
+                       help='Path to the output directory for html reports')
+    parser.add_argument('binaries', metavar='B', type=str, nargs='*',
+                       help='Path to an instrumented binary')
+    parser.add_argument('--only-merge', action='store_true',
+                        help='Only merge raw profiles together, skip report '
+                             'generation')
+    parser.add_argument('--preserve-profiles',
+                       help='Do not delete raw profiles', action='store_true')
+    parser.add_argument('--use-existing-profdata',
+                       help='Specify an existing indexed profile to use')
+    parser.add_argument('--unified-report', action='store_true',
+                       help='Emit a unified report for all binaries')
+    parser.add_argument('--restrict', metavar='R', type=str, nargs='*',
+                       default=[],
+                       help='Restrict the reporting to the given source paths'
+                   ' (must be specified after all other positional arguments)')
+    parser.add_argument('-C', '--compilation-dir', type=str, default="",
+                       help='The compilation directory of the binary')
+    args = parser.parse_args()
+
+    if args.use_existing_profdata and args.only_merge:
+        print('--use-existing-profdata and --only-merge are incompatible')
+        exit(1)
+
+    if args.use_existing_profdata:
+        profdata_path = args.use_existing_profdata
+    else:
+        profdata_path = merge_raw_profiles(args.host_llvm_profdata,
+                                           args.profile_data_dir,
+                                           args.preserve_profiles)
+
+    if not len(args.binaries):
+        print('No binaries specified, no work to do!')
+        exit(1)
+
+    if not args.only_merge:
+        prepare_html_reports(args.host_llvm_cov, profdata_path, args.report_dir,
+                             args.binaries, args.unified_report, args.restrict, 
+                             args.compilation_dir)