DirectXShaderCompiler/utils/hct/VerifierHelper.py

# Copyright (C) Microsoft Corporation. All rights reserved.
# This file is distributed under the University of Illinois Open Source License. See LICENSE.TXT for details.
r"""VerifierHelper.py - help with test content used with:
    clang-hlsl-tests /name:VerifierTest.*

    This script will produce an HLSL file with expected-error and expected-warning
    statements corresponding to actual errors/warnings produced from clang-hlsl-tests.
    The new file will be located in %TEMP%, named after the original file, but with
    the added extension '.result'.
    This can then be compared with the original file (such as varmods-syntax.hlsl)
    to see the differences in errors.  It may also be used to replace the original
    file, once the correct output behavior is verified.

    This script can also be used to do the same with fxc, adding expected errors there too.
    If there were errors/warnings/notes reported by clang, but nothing reported by fxc, an
    "fxc-pass {{}}" entry will be added.  If copied to reference, it means that you sign
    off on the difference in behavior between clang and fxc.

    In ast mode, this will find the ast subtree corresponding to a line of code preceding
    a line containing only: "/*verify-ast", and insert a stripped subtree between this marker
    and a line containing only: "*/".  This relies on clang.exe in the build directory.

    This tool expects clang.exe and clang-hlsl-tests.dll to be in %HLSL_BLD_DIR%\bin\Debug.

Usage:
    VerifierHelper.py clang <testname>  - run test through clang-hlsl-tests and show differences
    VerifierHelper.py fxc <testname>    - run test through fxc and show differences
    VerifierHelper.py ast <testname>    - run test through ast-dump and show differences
    VerifierHelper.py all <testname>    - run test through clang-hlsl-tests, ast-dump, and fxc, then show differences
<testname> - name of verifier test as passed to "te clang-hlsl-tests.dll /name:VerifierTest::<testname>":
    Example: RunVarmodsSyntax
    Can also specify * to run all tests

Environment variables - set these to ensure this tool works properly:
    HLSL_SRC_DIR        - root path of HLSLonLLVM enlistment
    HLSL_BLD_DIR        - path to projects and build output
    HLSL_FXC_PATH       - fxc.exe to use for comparison purposes
    HLSL_DIFF_TOOL      - tool to use for file comparison (optional)
"""

import os, sys, re

try:    DiffTool = os.environ['HLSL_DIFF_TOOL']
except: DiffTool = None
try:    FxcPath = os.environ['HLSL_FXC_PATH']
except: FxcPath = 'fxc'
HlslVerifierTestCpp = os.path.expandvars(r'${HLSL_SRC_DIR}\tools\clang\unittests\HLSL\VerifierTest.cpp')
HlslDataDir = os.path.expandvars(r'${HLSL_SRC_DIR}\tools\clang\test\HLSL')
HlslBinDir = os.path.expandvars(r'${HLSL_BLD_DIR}\Debug\bin')
VerifierTests = {
    'RunAttributes': 'attributes.hlsl',
    'RunBadInclude': 'bad-include.hlsl',
    'RunBinopDims': 'binop-dims.hlsl',
    'RunCXX11Attributes': 'cxx11-attributes.hlsl',
    'RunConstAssign': 'const-assign.hlsl',
    'RunConstDefault': 'const-default.hlsl',
    'RunConstExpr': 'const-expr.hlsl',
    'RunCppErrors': 'cpp-errors.hlsl',
    'RunCppErrorsHV2015': 'cpp-errors-hv2015.hlsl',
    'RunDerivedToBaseCasts': 'derived-to-base.hlsl',
    'RunEffectsSyntax': 'effects-syntax.hlsl',
    'RunEnums': 'enums.hlsl',
    'RunFunctions': 'functions.hlsl',
    'RunImplicitCasts': 'implicit-casts.hlsl',
    'RunIndexingOperator': 'indexing-operator.hlsl',
    'RunIntrinsicExamples': 'intrinsic-examples.hlsl',
    'RunLiterals': 'literals.hlsl',
    'RunMatrixAssignments': 'matrix-assignments.hlsl',
    'RunMatrixSyntax': 'matrix-syntax.hlsl',
    'RunMatrixSyntaxExactPrecision': 'matrix-syntax-exact-precision.hlsl',
    'RunMoreOperators': 'more-operators.hlsl',
    'RunObjectOperators': 'object-operators.hlsl',
    'RunPackReg': 'packreg.hlsl',
    'RunRayTracings': "raytracing.hlsl",
    'RunScalarAssignments': 'scalar-assignments.hlsl',
    'RunScalarAssignmentsExactPrecision': 'scalar-assignments-exact-precision.hlsl',
    'RunScalarOperators': 'scalar-operators.hlsl',
    'RunScalarOperatorsAssign': 'scalar-operators-assign.hlsl',
    'RunScalarOperatorsAssignExactPrecision': 'scalar-operators-assign-exact-precision.hlsl',
    'RunScalarOperatorsExactPrecision': 'scalar-operators-exact-precision.hlsl',
    'RunSemantics': 'semantics.hlsl',
    'RunString': 'string.hlsl',
    'RunStructAssignments': 'struct-assignments.hlsl',
    'RunTemplateChecks': 'template-checks.hlsl',
    'RunTypemodsSyntax': 'typemods-syntax.hlsl',
    'RunUint4Add3': 'uint4_add3.hlsl',
    'RunVarmodsSyntax': 'varmods-syntax.hlsl',
    'RunVectorAssignments': 'vector-assignments.hlsl',
    'RunVectorConditional': 'vector-conditional.hlsl',
    'RunVectorSyntax': 'vector-syntax.hlsl',
    'RunVectorSyntaxExactPrecision': 'vector-syntax-exact-precision.hlsl',
    'RunVectorSyntaxMix': 'vector-syntax-mix.hlsl',
    'RunWave': 'wave.hlsl',
}

# The following test(s) do not work in fxc mode:
fxcExcludedTests = [
    'RunCppErrors',
    'RunCppErrorsHV2015',
    'RunCXX11Attributes',
    'RunEnums',
    'RunIntrinsicExamples',
    'RunMatrixSyntaxExactPrecision',
    'RunRayTracings',
    'RunScalarAssignmentsExactPrecision',
    'RunScalarOperatorsAssignExactPrecision',
    'RunScalarOperatorsExactPrecision',
    'RunVectorSyntaxExactPrecision',
    'RunWave',
]

# rxRUN = re.compile(r'[ RUN      ] VerifierTest.(\w+)')	# gtest syntax
rxRUN = re.compile(r'StartGroup: VerifierTest::(\w+)')		# TAEF syntax
rxEndGroup = re.compile(r'EndGroup: VerifierTest::(\w+)\s+\[(\w+)\]')      # TAEF syntax
rxForProgram = re.compile(r'^for program (.*?) with errors\:$')
# rxExpected = re.compile(r"^error\: \'(\w+)\' diagnostics (expected but not seen|seen but not expected)\: $")  # gtest syntax
rxExpected = re.compile(r"^\'(\w+)\' diagnostics (expected but not seen|seen but not expected)\: $")    # TAEF syntax
rxDiagReport = re.compile(r'  (?:File (.*?) )?Line (\d+): (.*)$')

rxDiag = re.compile(r'((expected|fxc)-(error|warning|note|pass)\s*\{\{(.*?)\}\}\s*)')

rxFxcErr = re.compile(r'(.+)\((\d+)(?:,(\d+)(?:\-(\d+))?)?\)\: (error|warning) (.*?)\: (.*)')
# groups = (filename, line, colstart, colend, ew, error_code, error_message)

rxCommentStart = re.compile(r'(//|/\*)')
rxStrings = re.compile(r'(\'|\").*?((?<!\\)\1)')
rxBraces = re.compile(r'(\(|\)|\{|\}|\[|\])')
rxStatementEndOrBlockBegin = re.compile(r'(\;|\{)')
rxLineContinued = re.compile(r'.*\\$')
rxVerifyArguments = re.compile(r'\s*//\s*\:FXC_VERIFY_ARGUMENTS\:\s+(.*)')

rxVerifierTestMethod = re.compile(r'TEST_F\(VerifierTest,\s*(\w+)\)\s*')
rxVerifierTestCheckFile = re.compile(r'CheckVerifiesHLSL\s*\(\s*L?\"([^"]+)"\s*\)')

rxVerifyAst = re.compile(r'^\s*(\/\*verify\-ast)\s*$')    # must start with line containing only "/*verify-ast"
rxEndVerifyAst = re.compile(r'^\s*\*\/\s*$')             # ends with line containing only "*/"
rxAstSourceLocation = re.compile(
    r'''\<(?:(?P<Invalid>\<invalid\ sloc\>) |
             (?:
                (?:(?:(?P<FromFileLine>line|\S*):(?P<FromLine>\d+):(?P<FromLineCol>\d+)) |
                   col:(?P<FromCol>\d+)
                )
                (?:,\s+
                   (?:(?:(?P<ToFileLine>line|\S*):(?P<ToLine>\d+):(?P<ToLineCol>\d+)) |
                      col:(?P<ToCol>\d+)
                   )
                )?
             )
          )\>''',
    re.VERBOSE)
rxAstHexAddress = re.compile(r'\b(0x[0-9a-f]+) ?')
rxAstNode = re.compile(r'((?:\<\<\<NULL\>\>\>)|(?:\w+))\s*(.*)')
# matches ignored portion of line for first AST node in subgraph to match
rxAstIgnoredIndent = re.compile(r'^(\s+|\||\`|\-)*')

# The purpose of StripComments and CountBraces is to be used when commenting lines of code out to allow
# Fxc testing to continue even when it doesn't recover as well as clang.  Some error lines are on the
# beginning of a function, where commenting just that line will comment out the beginning of the function
# block, but not the body or end of the block, producing invalid syntax.  Here's an example:
#   void foo(error is here) { /* expected-error {{some expected clang error}} */
#     return;
#   }
# If the first line is commented without the rest of the function, it will be incorrect code.
# So the intent is to detect when the line being commented out results in an unbalanced brace matching.
# Then these functions will be used to comment additional lines until the braces match again.
# It's simple and won't handle the general case, but should handle the cases in the test files, and if
# not, the tests should be easily modifyable to work with it.
# This still does not handle preprocessor directives, or escaped characters (like line ends or escaped
# quotes), or other cases that a real parser would handle.

def StripComments(line, multiline_comment_continued = False):
    "Remove comments from line, returns stripped line and multiline_comment_continued if a multiline comment continues beyond the line"
    if multiline_comment_continued:
        # in multiline comment, only look for end of that
        idx = line.find('*/')
        if idx < 0:
            return '', True
        return StripComments(line[idx+2:])
    # look for start of multiline comment or eol comment:
    m = rxCommentStart.search(line)
    if m:
        if m.group(1) == '/*':
            line_end, multiline_comment_continued = StripComments(line[m.end(1):], True)
            return line[:m.start(1)] + line_end, multiline_comment_continued
        elif m.group(1) == '//':
            return line[:m.start(1)], False
    return line, False

def CountBraces(line, bracestacks):
    m = rxStrings.search(line)
    if m:
        CountBraces(line[:m.start(1)], bracestacks)
        CountBraces(line[m.end(2):], bracestacks)
        return
    for b in rxBraces.findall(line):
        if b in '()':
            bracestacks['()'] = bracestacks.get('()', 0) + ((b == '(') and 1 or -1)
        elif b in '{}':
            bracestacks['{}'] = bracestacks.get('{}', 0) + ((b == '{') and 1 or -1)
        elif b in '[]':
            bracestacks['[]'] = bracestacks.get('[]', 0) + ((b == '[') and 1 or -1)

def ProcessStatementOrBlock(lines, start, fn_process):
    num = 0
    # statement_continued initialized with whether line has non-whitespace content
    statement_continued = not not StripComments(lines[start], False)[0].strip()
    # Assumes start of line is not inside multiline comment
    multiline_comment_continued = False
    bracestacks = {}
    while start+num < len(lines):
        line = lines[start+num]
        lines[start+num] = fn_process(line)
        num += 1
        line, multiline_comment_continued = StripComments(line, multiline_comment_continued)
        CountBraces(line, bracestacks)
        if (statement_continued and 
            not rxStatementEndOrBlockBegin.search(line) ):
            continue
        statement_continued = False
        if rxLineContinued.match(line):
            continue
        if (bracestacks.get('{}', 0) < 1 and
            bracestacks.get('()', 0) < 1 and
            bracestacks.get('[]', 0) < 1 ):
            break
    return num

def CommentStatementOrBlock(lines, start):
    def fn_process(line):
        return '//  ' + line
    return ProcessStatementOrBlock(lines, start, fn_process)

def ParseVerifierTestCpp():
    "Returns dictionary mapping Run* test name to hlsl filename by parsing VerifierTest.cpp"
    tests = {}
    FoundTest = None
    def fn_null(line):
        return line
    def fn_process(line):
        searching = FoundTest is not None
        if searching:
            m = rxVerifierTestCheckFile.search(line)
            if m:
                tests[FoundTest] = m.group(1)
                searching = False
        return line
    with open(HlslVerifierTestCpp, 'rt') as f:
        lines = f.readlines()
        start = 0
        while start < len(lines):
            m = rxVerifierTestMethod.search(lines[start])
            if m:
                FoundTest = m.group(1)
                start += ProcessStatementOrBlock(lines, start, fn_process)
                if FoundTest not in tests:
                    print('Could not parse file for test %s' % FoundTest)
                FoundTest = None
            else:
                start += ProcessStatementOrBlock(lines, start, fn_null)
    return tests

class SourceLocation(object):
    def __init__(self, line=None, **kwargs):
        if not kwargs:
            self.Invalid = '<invalid sloc>'
            return
        for key, value in kwargs.items():
            try:    value = int(value)
            except: pass
            setattr(self, key, value)
        if line and not self.FromLine:
            self.FromLine = line
        self.FromCol = self.FromCol or self.FromLineCol
        self.ToCol = self.ToCol or self.ToLineCol
    def Offset(self, offset):
        "Offset From/To Lines by specified value"
        if self.Invalid:
            return
        if self.FromLine:
            self.FromLine = self.FromLine + offset
        if self.ToLine:
            self.ToLine = self.ToLine + offset
    def ToStringAtLine(self, line):
        "convert to string relative to specified line"
        if self.Invalid:
            sloc = self.Invalid
        else:
            if self.FromLine and line != self.FromLine:
                sloc = 'line:%d:%d' % (self.FromLine, self.FromCol)
                line = self.FromLine
            else:
                sloc = 'col:%d' % self.FromCol
            if self.ToCol:
                if self.ToLine and line != self.ToLine:
                    sloc += ', line:%d:%d' % (self.ToLine, self.ToCol)
                else:
                    sloc += ', col:%d' % self.ToCol
        return '<' + sloc + '>'

class AstNode(object):
    def __init__(self, name, sloc, prefix, text, indent=''):
        self.name, self.sloc, self.prefix, self.text, self.indent = name, sloc, prefix, text, indent
        self.children = []
    def ToStringAtLine(self, line):
        "convert to string relative to specified line"
        if self.name == '<<<NULL>>>':
            return self.name
        return ('%s %s%s %s' % (self.name, self.prefix, self.sloc.ToStringAtLine(line), self.text)).strip()

def WalkAstChildren(ast_root):
    "yield each child node in the ast tree in depth-first order"
    for node in ast_root.children:
        yield node
        for child in WalkAstChildren(node):
            yield child

def WriteAstSubtree(ast_root, line, indent=''):
    output = []
    output.append(indent + ast_root.ToStringAtLine(line))
    if not ast_root.sloc.Invalid and ast_root.sloc.FromLine:
        line = ast_root.sloc.FromLine
    root_indent_len = len(ast_root.indent)
    for child in WalkAstChildren(ast_root):
        output.append(indent + child.indent[root_indent_len:] + child.ToStringAtLine(line))
        if not child.sloc.Invalid and child.sloc.FromLine:
            line = child.sloc.FromLine
    return output

def FindAstNodesByLine(ast_root, line):
    nodes = []
    if not ast_root.sloc.Invalid and ast_root.sloc.FromLine == line:
        return [ast_root]
    if not ast_root.sloc.Invalid and ast_root.sloc.ToLine and ast_root.sloc.ToLine < line:
        return []
    for child in ast_root.children:
        sub_nodes = FindAstNodesByLine(child, line)
        if sub_nodes:
            nodes += sub_nodes
    return nodes

def ParseAst(astlines):
    cur_line = 0    # current source line
    root_node = None
    ast_stack = []  # stack of nodes and column numbers so we can pop the right number of nodes up the stack
    i = 0   # ast line index

    def push(node, col):
        if ast_stack:
            cur_node, prior_col = ast_stack[-1]
            cur_node.children.append(node)
        ast_stack.append((node, col))
    def popto(col):
        cur_node, prior_col = ast_stack[-1]
        while ast_stack and col <= prior_col:
            ast_stack.pop()
            cur_node, prior_col = ast_stack[-1]
        assert ast_stack
    def parsenode(text, indent):
        m = rxAstNode.match(text)
        if m:
            name = m.group(1)
            text = text[m.end(1):].strip()
        else:
            print('rxAstNode match failed on:\n  %s' % text)
            return AstNode('ast-parse-failed', SourceLocation(), '', '', indent)
        text = rxAstHexAddress.sub('', text).strip()
        m = rxAstSourceLocation.search(text)
        if m:
            prefix = text[:m.start()]
            sloc = SourceLocation(cur_line, **m.groupdict())
            text = text[m.end():].strip()
        else:
            prefix = ''
            sloc = SourceLocation()
        return AstNode(name, sloc, prefix, text, indent)

    # Look for TranslationUnitDecl and start from there
    while i < len(astlines):
        text = astlines[i]
        if text.startswith('TranslationUnitDecl'):
            root_node = parsenode(text, '')
            push(root_node, 0)
            break
        i += 1
    i += 1

    # gather ast nodes
    while i < len(astlines):
        line = astlines[i]

        # get starting column and update stack
        m = rxAstIgnoredIndent.match(line)
        indent = ''
        col = 0
        if m:
            indent = m.group(0)
            col = m.end()
        if col == 0:
            break   # at this point we should be done parsing the translation unit!

        popto(col)

        # parse and add the node
        node = parsenode(line[col:], indent)
        if not node:
            print('error parsing line %d:\n%s' % (i+1, line))
            assert False
        push(node, col)

        # update current source line
        sloc = node.sloc
        if not sloc.Invalid and sloc.FromLine:
            cur_line = sloc.FromLine

        i += 1

    return root_node

class File(object):
    def __init__(self, filename):
        self.filename = filename
        self.expected = {}      # {line_num: [('error' or 'warning', 'error or warning message'), ...], ...}
        self.unexpected = {}    # {line_num: [('error' or 'warning', 'error or warning message'), ...], ...}
        self.last_diag_col = None
    def AddExpected(self, line_num, ew, message):
        self.expected.setdefault(line_num, []).append((ew, message))
    def AddUnexpected(self, line_num, ew, message):
        self.unexpected.setdefault(line_num, []).append((ew, message))

    def MatchDiags(self, line, diags=[], prefix='expected', matchall=False):
        diags = diags[:]
        diag_col = None
        matches = []
        for m in rxDiag.finditer(line):
            if diag_col is None:
                diag_col = m.start()
                self.last_diag_col = diag_col
            if m.group(2) == prefix:
                pattern = m.groups()[2:4]
                for idx, (ew, message) in enumerate(diags):
                    if pattern == (ew, message):
                        matches.append(m)
                        break
                else:
                    if matchall:
                        matches.append(m)
                    continue
                del diags[idx]
        return sorted(matches, key=lambda m: m.start()), diags, diag_col
    def RemoveDiags(self, line, diags, prefix='expected', removeall=False):
        """Removes expected-* diags from line, returns result_line, remaining_diags, diag_col
        Where, result_line is the line without the matching diagnostics,
            remaining is the list of diags not found on the line,
            diag_col is the column of the first diagnostic found on the line.
        """
        matches, diags, diag_col = self.MatchDiags(line, diags, prefix, removeall)
        for m in reversed(matches):
            line = line[:m.start()] + line[m.end():]
        return line, diags, diag_col
    def AddDiags(self, line, diags, diag_col=None, prefix='expected'):
        "Adds expected-* diags to line."
        if diags:
            if diag_col is None:
                if self.last_diag_col is not None and self.last_diag_col-3 > len(line):
                    diag_col = self.last_diag_col
                else:
                    diag_col = max(len(line) + 7, 63) # 4 spaces + '/* ' or at column 63, whichever is greater
                line = line + (' ' * ((diag_col - 3) - len(line))) + '/* */'
            for ew, message in reversed(diags):
                line = line[:diag_col] + ('%s-%s {{%s}} ' % (prefix, ew, message)) + line[diag_col:]
        return line.rstrip()
    def SortDiags(self, line):
        matches = list(rxDiag.finditer(line))
        if matches:
            for m in sorted(matches, key=lambda m: m.start(), reverse=True):
                line = line[:m.start()] + line[m.end():]
                diag_col = m.start()
            for m in sorted(matches, key=lambda m: m.groups()[1:], reverse=True):
                line = line[:diag_col] + ('%s-%s {{%s}} ' % m.groups()[1:]) + line[diag_col:]
        return line.rstrip()

    def OutputResult(self):
        temp_filename = os.path.expandvars(r'${TEMP}\%s' % os.path.split(self.filename)[1])
        with open(self.filename, 'rt') as fin:
            with open(temp_filename+'.result', 'wt') as fout:
                line_num = 0
                for line in fin.readlines():
                    if line[-1] == '\n':
                        line = line[:-1]
                    line_num += 1
                    line, expected, diag_col = self.RemoveDiags(line, self.expected.get(line_num, []))
                    for ew, message in expected:
                        print('Error: Line %d: Could not find: expected-%s {{%s}}!!' % (line_num, ew, message))
                    line = self.AddDiags(line, self.unexpected.get(line_num, []), diag_col)
                    line = self.SortDiags(line)
                    fout.write(line + '\n')

    def TryFxc(self, result_filename=None):
        temp_filename = os.path.expandvars(r'${TEMP}\%s' % os.path.split(self.filename)[1])
        if result_filename is None:
            result_filename = temp_filename + '.fxc'
        inlines = []
        with open(self.filename, 'rt') as fin:
            for line in fin.readlines():
                if line[-1] == '\n':
                    line = line[:-1]
                inlines.append(line)
        verify_arguments = None
        for line in inlines:
            m = rxVerifyArguments.search(line)
            if m:
                verify_arguments = m.group(1)
                print('Found :FXC_VERIFY_ARGUMENTS: %s' % verify_arguments)
                break

        # result will hold the final result after adding fxc error messages
        # initialize it by removing all the expected diagnostics
        result = [(line, None, False) for line in inlines]
        for n, (line, diag_col, expected) in enumerate(result):
            line, diags, diag_col = self.RemoveDiags(line, [], prefix='fxc', removeall=True)
            matches, diags, diag_col2 = self.MatchDiags(line, [], prefix='expected', matchall=True)
            if matches:
                expected = True
##                if diag_col is None:
##                    diag_col = diag_col2
##                elif diag_col2 < diag_col:
##                    diag_col = diag_col2
            result[n] = (line, diag_col, expected)

        # commented holds the version that gets progressively commented as fxc reports errors
        commented = inlines[:]

        # diags_by_line is a dictionary of a set of errors and warnings keyed off line_num
        diags_by_line = {}
        while True:
            with open(temp_filename+'.fxc_temp', 'wt') as fout:
                fout.write('\n'.join(commented))
                if verify_arguments is None:
                    fout.write("\n[numthreads(1,1,1)] void _test_main() {  }\n")
            if verify_arguments is None:
                args = '/E _test_main /T cs_5_1'
            else:
                args = verify_arguments
            os.system('%s /nologo "%s.fxc_temp" %s /DVERIFY_FXC=1 /Fo "%s.fxo" /Fe "%s.err" 1> "%s.log" 2>&1' % 
                      (FxcPath, temp_filename, args, temp_filename, temp_filename, temp_filename))
            with open(temp_filename+'.err', 'rt') as f:
                errors = [m for m in map(rxFxcErr.match, f.readlines()) if m]
            errors = sorted(errors, key=lambda m: int(m.group(2)))
            first_error = None
            for m in errors:
                line_num = int(m.group(2))
                if not first_error and m.group(5) == 'error':
                    first_error = line_num
                elif first_error and line_num > first_error:
                    break
                diags_by_line.setdefault(line_num, set()).add((m.group(5), m.group(6) + ': ' + m.group(7)))
            if first_error and first_error <= len(commented):
                CommentStatementOrBlock(commented, first_error-1)
            else:
                break

        # Add diagnostic messages from fxc to result:
        self.last_diag_col = None
        for i, (line, diag_col, expected) in enumerate(result):
            line_num = i + 1
            if diag_col:
                self.last_diag_col = diag_col
            diags = diags_by_line.get(line_num, set())
            if not diags:
                if expected:
                    diags.add(('pass', ''))
                else:
                    continue
            diags = sorted(list(diags))
            line = self.SortDiags(self.AddDiags(line, diags, diag_col, prefix='fxc'))
            result[i] = line, diag_col, expected

        with open(result_filename, 'wt') as f:
            f.write('\n'.join(map(lambda (line, diag_col, expected): line, result)))

    def TryAst(self, result_filename=None):
        temp_filename = os.path.expandvars(r'${TEMP}\%s' % os.path.split(self.filename)[1])
        if result_filename is None:
            result_filename = temp_filename + '.ast'
        try:    os.unlink(temp_filename+'.ast_dump')
        except: pass
        try:    os.unlink(result_filename)
        except: pass
##        result = os.system('%s\\clang.exe -cc1 -fsyntax-only -ast-dump %s 1>"%s.ast_dump" 2>"%s.log"' %
        result = os.system('%s\\dxc.exe -ast-dump %s -E main -T ps_5_0 1>"%s.ast_dump" 2>"%s.log"' %
                           (HlslBinDir, self.filename, temp_filename, temp_filename))
        # dxc dumps ast even if there exists any syntax error. If there is any error, dxc returns some nonzero errorcode.
        if not os.path.isfile(temp_filename+'.ast_dump'):
            print('ast-dump failed, see log:\n  %s.log' % (temp_filename))
            return
##        elif result:
##            print('ast-dump succeeded, but exited with error code %d, see log:\n  %s.log' % (result, temp_filename))
        astlines = []
        with open(temp_filename+'.ast_dump', 'rt') as fin:
            for line in fin.readlines():
                if line[-1] == '\n':
                    line = line[:-1]
                astlines.append(line)
        try:
            ast_root = ParseAst(astlines)
        except:
            print('ParseAst failed on "%s"' % (temp_filename + '.ast_dump'))
            raise
        inlines = []
        with open(self.filename, 'rt') as fin:
            for line in fin.readlines():
                if line[-1] == '\n':
                    line = line[:-1]
                inlines.append(line)
        outlines = []
        i = 0
        while i < len(inlines):
            line = inlines[i]
            outlines.append(line)
            m = rxVerifyAst.match(line)
            if m:
                indent = line[:m.start(1)] + '  '
                # at this point i is the ONE based source line number
                # (since it's one past the line we want to verify in zero based index)
                ast_nodes = FindAstNodesByLine(ast_root, i)
                if not ast_nodes:
                    outlines += [indent + 'No matching AST found for line!']
                else:
                    for ast in ast_nodes:
                        outlines += WriteAstSubtree(ast, i, indent)
                while i+1 < len(inlines) and not rxEndVerifyAst.match(inlines[i+1]):
                    i += 1
            i += 1

        with open(result_filename, 'wt') as f:
            f.write('\n'.join(outlines))

def ProcessVerifierOutput(lines):
    files = {}
    cur_filename = None
    cur_test = None
    state = 'WaitingForFile'
    ew = ''
    expected = None
    for line in lines:
        if not line:
            continue
        if line[-1] == '\n':
            line = line[:-1]
        m = rxRUN.match(line)
        if m:
            cur_test = m.group(1)
        m = rxForProgram.match(line)
        if m:
            cur_filename = m.group(1)
            files[cur_filename] = File(cur_filename)
            state = 'WaitingForCategory'
            continue
        if state is 'WaitingForFile':
            m = rxEndGroup.match(line)
            if m and m.group(2) == 'Failed':
                # This usually happens when compiler crashes
                print('Fatal Error: test %s failed without verifier results.' % cur_test)
        if state is 'WaitingForCategory' or state is 'ReadingErrors':
            m = rxExpected.match(line)
            if m:
                ew = m.group(1)
                expected = m.group(2) == 'expected but not seen'
                state = 'ReadingErrors'
                continue
        if state is 'ReadingErrors':
            m = rxDiagReport.match(line)
            if m:
                line_num = int(m.group(2))
                if expected:
                    files[cur_filename].AddExpected(line_num, ew, m.group(3))
                else:
                    files[cur_filename].AddUnexpected(line_num, ew, m.group(3))
                continue
    for f in files.values():
        f.OutputResult()
    return files


def maybe_compare(filename1, filename2):
    with open(filename1, 'rt') as fbefore:
        with open(filename2, 'rt') as fafter:
            before = fbefore.read()
            after = fafter.read()
    if before.strip() != after.strip():
        print('Differences found.  Compare:\n  %s\nwith:\n  %s' % (filename1, filename2))
        if DiffTool:
            os.system('%s %s %s' % (DiffTool, filename1, filename2))
        return True
    return False

def PrintUsage():
    print(__doc__)
    print('Available tests and corresponding files:')
    tests = sorted(VerifierTests.keys())
    width = len(max(tests, key=len))
    for name in tests:
        print(('    %%-%ds  %%s' % width) % (name, VerifierTests[name]))
    print('Tests incompatible with fxc mode:')
    for name in fxcExcludedTests:
        print('    %s' % name)

def RunVerifierTest(test, HlslDataDir=HlslDataDir):
    import codecs
    temp_filename = os.path.expandvars(r'${TEMP}\VerifierHelper_temp.txt')
    cmd = ('te %s\\clang-hlsl-tests.dll /p:"HlslDataDir=%s" /name:VerifierTest::%s > %s' %
           (HlslBinDir, HlslDataDir, test, temp_filename))
    print(cmd)
    os.system(cmd)      # TAEF test
    # TAEF outputs unicode, so read as binary and convert:
    with open(temp_filename, 'rb') as f:
        return codecs.decode(f.read(), 'UTF-16').replace(u'\x7f', u'').replace(u'\r\n', u'\n').splitlines()

def main(*args):
    global VerifierTests
    try:
        VerifierTests = ParseVerifierTestCpp()
    except:
        print('Unable to parse tests from VerifierTest.cpp; using defaults')
    if len(args) < 1 or (args[0][0] in '-/' and args[0][1:].lower() in ('h', '?', 'help')):
        PrintUsage()
        return -1
    mode = args[0]
    if mode == 'fxc':
        allFxcTests = sorted(filter(lambda key: key not in fxcExcludedTests, VerifierTests.keys()))
        if args[1] == '*':
            tests = allFxcTests
        else:
            if args[1] not in allFxcTests:
                PrintUsage()
                return -1
            tests = [args[1]]
        differences = False
        for test in tests:
            print('---- %s ----' % test)
            filename = os.path.join(HlslDataDir, VerifierTests[test])
            result_filename = os.path.expandvars(r'${TEMP}\%s.fxc' % os.path.split(filename)[1])
            File(filename).TryFxc()
            differences = maybe_compare(filename, result_filename) or differences
        if not differences:
            print('No differences found!')
    elif mode == 'clang':
        if args[1] != '*' and args[1] not in VerifierTests:
            PrintUsage()
            return -1
        files = ProcessVerifierOutput(RunVerifierTest(args[1]))
        differences = False
        if files:
            for f in files.values():
                if f.expected or f.unexpected:
                    result_filename = os.path.expandvars(r'${TEMP}\%s.result' % os.path.split(f.filename)[1])
                    differences = maybe_compare(f.filename, result_filename) or differences
        if not differences:
            print('No differences found!')
    elif mode == 'ast':
        allAstTests = sorted(VerifierTests.keys())
        if args[1] == '*':
            tests = allAstTests
        else:
            if args[1] not in allAstTests:
                PrintUsage()
                return -1
            tests = [args[1]]
        differences = False
        for test in tests:
            print('---- %s ----' % test)
            filename = os.path.join(HlslDataDir, VerifierTests[test])
            result_filename = os.path.expandvars(r'${TEMP}\%s.ast' % os.path.split(filename)[1])
            File(filename).TryAst()
            differences = maybe_compare(filename, result_filename) or differences
        if not differences:
            print('No differences found!')
    elif mode == 'all':
        allTests = sorted(VerifierTests.keys())
        if args[1] == '*':
            tests = allTests
        else:
            if args[1] not in allTests:
                PrintUsage()
                return -1
            tests = [args[1]]

        # Do clang verifier tests, updating source file paths for changed files:
        sourceFiles = dict([(VerifierTests[test], os.path.join(HlslDataDir, VerifierTests[test])) for test in tests])
        files = ProcessVerifierOutput(RunVerifierTest(args[1]))
        if files:
            for f in files.values():
                if f.expected or f.unexpected:
                    name = os.path.split(f.filename)[1]
                    sourceFiles[name] = os.path.expandvars(r'${TEMP}\%s.result' % name)

        # update verify-ast blocks:
        for name, sourceFile in sourceFiles.items():
            result_filename = os.path.expandvars(r'${TEMP}\%s.ast' % name)
            File(sourceFile).TryAst(result_filename)
            sourceFiles[name] = result_filename

        # now do fxc verification and final comparison
        differences = False
        fxcExcludedFiles = [VerifierTests[test] for test in fxcExcludedTests]
        width = len(max(tests, key=len))
        for test in tests:
            name = VerifierTests[test]
            sourceFile = sourceFiles[name]
            print(('Test %%-%ds - %%s' % width) % (test, name))
            result_filename = os.path.expandvars(r'${TEMP}\%s.fxc' % name)
            if name not in fxcExcludedFiles:
                File(sourceFile).TryFxc(result_filename)
                sourceFiles[name] = result_filename
            differences = maybe_compare(os.path.join(HlslDataDir, name), sourceFiles[name]) or differences
        if not differences:
            print('No differences found!')
    else:
        PrintUsage()
        return -1

    return 0

if __name__ == '__main__':
    sys.exit(main(*sys.argv[1:]))