Fix: improve supports for source position tracking

src/Lua/CodeAnalysis/Compilation/LuaCompiler.cs

@@ -21,6 +21,8 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
     {
         using var context = FunctionCompilationContext.Create(null);
         context.HasVariableArguments = true;
+        context.LineDefined = syntaxTree.Position.Line;
+        context.LastLineDefined = syntaxTree.Position.Line;
         // set global enviroment upvalue
         context.AddUpValue(new()
         {
@@ -586,10 +588,10 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
     // function declaration
     public bool VisitFunctionDeclarationExpressionNode(FunctionDeclarationExpressionNode node, ScopeCompilationContext context)
     {
-        var funcIndex = CompileFunctionProto(ReadOnlyMemory<char>.Empty, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length, node.HasVariableArguments, false, node.LineDefined, node.LastLineDefined);
+        var funcIndex = CompileFunctionProto(ReadOnlyMemory<char>.Empty, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length, node.HasVariableArguments, false, node.LineDefined, node.EndPosition.Line);
 
         // push closure instruction
-        context.PushInstruction(Instruction.Closure(context.StackPosition, funcIndex), node.Position, true);
+        context.PushInstruction(Instruction.Closure(context.StackPosition, funcIndex), node.EndPosition, true);
 
         return true;
     }
@@ -604,23 +606,23 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
         });
 
         // compile function
-        var funcIndex = CompileFunctionProto(node.Name, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length, node.HasVariableArguments, false, node.LineDefined, node.LastLineDefined);
+        var funcIndex = CompileFunctionProto(node.Name, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length, node.HasVariableArguments, false, node.LineDefined, node.EndPosition.Line);
 
         // push closure instruction
-        context.PushInstruction(Instruction.Closure(context.StackPosition, funcIndex), node.Position, true);
+        context.PushInstruction(Instruction.Closure(context.StackPosition, funcIndex), node.EndPosition, true);
 
         return true;
     }
 
     public bool VisitFunctionDeclarationStatementNode(FunctionDeclarationStatementNode node, ScopeCompilationContext context)
     {
-        var funcIndex = CompileFunctionProto(node.Name, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length, node.HasVariableArguments, false, node.LineDefined, node.LastLineDefined);
+        var funcIndex = CompileFunctionProto(node.Name, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length, node.HasVariableArguments, false, node.LineDefined, node.EndPosition.Line);
 
         // add closure
         var index = context.Function.GetConstantIndex(node.Name.ToString());
 
         // push closure instruction
-        context.PushInstruction(Instruction.Closure(context.StackPosition, funcIndex), node.Position, true);
+        context.PushInstruction(Instruction.Closure(context.StackPosition, funcIndex), node.EndPosition, true);
 
         if (context.TryGetLocalVariableInThisScope(node.Name, out var variable))
         {
@@ -639,7 +641,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
     public bool VisitTableMethodDeclarationStatementNode(TableMethodDeclarationStatementNode node, ScopeCompilationContext context)
     {
         var funcIdentifier = node.MemberPath[^1];
-        var funcIndex = CompileFunctionProto(funcIdentifier.Name, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length + 1, node.HasVariableArguments, node.HasSelfParameter, node.LineDefined, node.LastLineDefined);
+        var funcIndex = CompileFunctionProto(funcIdentifier.Name, context, node.ParameterNodes, node.Nodes, node.ParameterNodes.Length + 1, node.HasVariableArguments, node.HasSelfParameter, node.LineDefined, node.EndPosition.Line);
 
         // add closure
         var index = context.Function.GetConstantIndex(funcIdentifier.Name.ToString());
@@ -660,7 +662,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
 
         // push closure instruction
         var closureIndex = context.StackPosition;
-        context.PushInstruction(Instruction.Closure(closureIndex, funcIndex), node.Position, true);
+        context.PushInstruction(Instruction.Closure(closureIndex, funcIndex), node.EndPosition, true);
 
         // set table
         context.PushInstruction(Instruction.SetTable(tableIndex, (ushort)(index + 256), closureIndex), funcIdentifier.Position);
@@ -758,10 +760,10 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
         // if
         using (var scopeContext = context.CreateChildScope())
         {
-            CompileConditionNode(node.IfNode.ConditionNode, scopeContext, true);
+            CompileConditionNode(node.IfNode.ConditionNode, scopeContext, true, node.IfNode.Position);
 
             var ifPosition = scopeContext.Function.Instructions.Length;
-            scopeContext.PushInstruction(Instruction.Jmp(0, 0), node.Position);
+            scopeContext.PushInstruction(Instruction.Jmp(0, 0), node.IfNode.Position);
 
             foreach (var childNode in node.IfNode.ThenNodes)
             {
@@ -772,7 +774,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
             if (hasElse)
             {
                 endJumpIndexList.Add(scopeContext.Function.Instructions.Length);
-                scopeContext.PushInstruction(Instruction.Jmp(stackPositionToClose, 0), node.Position, true);
+                scopeContext.PushInstruction(Instruction.Jmp(stackPositionToClose, 0), node.IfNode.ThenNodes[^1].Position, true);
             }
             else
             {
@@ -790,7 +792,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
             CompileConditionNode(elseIf.ConditionNode, scopeContext, true);
 
             var elseifPosition = scopeContext.Function.Instructions.Length;
-            scopeContext.PushInstruction(Instruction.Jmp(0, 0), node.Position);
+            scopeContext.PushInstruction(Instruction.Jmp(0, 0), elseIf.Position);
 
             foreach (var childNode in elseIf.ThenNodes)
             {
@@ -802,11 +804,11 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
             if (hasElse)
             {
                 endJumpIndexList.Add(scopeContext.Function.Instructions.Length);
-                scopeContext.PushInstruction(Instruction.Jmp(stackPositionToClose, 0), node.Position);
+                scopeContext.PushInstruction(Instruction.Jmp(stackPositionToClose, 0), elseIf.Position);
             }
             else
             {
-                scopeContext.TryPushCloseUpValue(stackPositionToClose, node.Position);
+                scopeContext.TryPushCloseUpValue(stackPositionToClose, elseIf.Position);
             }
 
             scopeContext.Function.Instructions[elseifPosition].SBx = scopeContext.Function.Instructions.Length - 1 - elseifPosition;
@@ -847,8 +849,9 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
 
         CompileConditionNode(node.ConditionNode, scopeContext, true);
         var a = scopeContext.HasCapturedLocalVariables ? (byte)(stackPosition + 1) : (byte)0;
-        scopeContext.PushInstruction(Instruction.Jmp(a, startIndex - scopeContext.Function.Instructions.Length - 1), node.Position);
-        scopeContext.TryPushCloseUpValue(scopeContext.StackPosition, node.Position);
+        var untilPosition = node.ConditionNode.Position;
+        scopeContext.PushInstruction(Instruction.Jmp(a, startIndex - scopeContext.Function.Instructions.Length - 1), untilPosition);
+        scopeContext.TryPushCloseUpValue(scopeContext.StackPosition, untilPosition);
 
         context.Function.LoopLevel--;
 
@@ -1018,7 +1021,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
 
         // push OP_TFORCALL and OP_TFORLOOP
         context.PushInstruction(Instruction.TForCall(startPosition, (ushort)node.Names.Length), node.Position);
-        context.PushInstruction(Instruction.TForLoop((byte)(startPosition + 2), startJumpIndex - context.Function.Instructions.Length), node.EndPosition);
+        context.PushInstruction(Instruction.TForLoop((byte)(startPosition + 2), startJumpIndex - context.Function.Instructions.Length), node.Position);
 
         context.Function.ResolveAllBreaks((byte)(startPosition + 1), context.Function.Instructions.Length - 1, scopeContext);
         context.StackPosition = startPosition;
@@ -1160,6 +1163,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
                                 value = -d1;
                                 return true;
                             }
+
                             break;
                         case UnaryOperator.Not:
                             if (unaryNodeValue.TryRead<bool>(out var b))
@@ -1167,9 +1171,11 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
                                 value = !b;
                                 return true;
                             }
+
                             break;
                     }
                 }
+
                 break;
             case BinaryExpressionNode binaryExpression:
                 if (TryGetConstant(binaryExpression.LeftNode, context, out var leftValue) &&
@@ -1234,7 +1240,8 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
     /// <param name="node">Condition node</param>
     /// <param name="context">Context</param>
     /// <param name="falseIsSkip">If true, generates an instruction sequence that skips the next instruction if the condition is false.</param>
-    void CompileConditionNode(ExpressionNode node, ScopeCompilationContext context, bool falseIsSkip)
+    /// <param name="testPosition">Position of the test instruction</param>
+    void CompileConditionNode(ExpressionNode node, ScopeCompilationContext context, bool falseIsSkip, SourcePosition? testPosition = null)
     {
         if (node is BinaryExpressionNode binaryExpression)
         {
@@ -1286,7 +1293,7 @@ public sealed class LuaCompiler : ISyntaxNodeVisitor<ScopeCompilationContext, bo
         }
 
         node.Accept(this, context);
-        context.PushInstruction(Instruction.Test((byte)(context.StackPosition - 1), falseIsSkip ? (byte)0 : (byte)1), node.Position);
+        context.PushInstruction(Instruction.Test((byte)(context.StackPosition - 1), falseIsSkip ? (byte)0 : (byte)1), testPosition ?? node.Position);
     }
 
     void CompileExpressionList(SyntaxNode rootNode, ExpressionNode[] expressions, int minimumCount, ScopeCompilationContext context)

src/Lua/CodeAnalysis/Syntax/LuaSyntaxTree.cs

@@ -1,6 +1,6 @@
 namespace Lua.CodeAnalysis.Syntax;
 
-public record LuaSyntaxTree(SyntaxNode[] Nodes) : SyntaxNode(new SourcePosition(0, 0))
+public record LuaSyntaxTree(SyntaxNode[] Nodes,SourcePosition Position) : SyntaxNode(Position)
 {
     public override TResult Accept<TContext, TResult>(ISyntaxNodeVisitor<TContext, TResult> visitor, TContext context)
     {

src/Lua/CodeAnalysis/Syntax/Nodes/FunctionDeclarationExpressionNode.cs

@@ -1,6 +1,6 @@
 namespace Lua.CodeAnalysis.Syntax.Nodes;
 
-public record FunctionDeclarationExpressionNode(IdentifierNode[] ParameterNodes, SyntaxNode[] Nodes, bool HasVariableArguments, SourcePosition Position, int LineDefined,int LastLineDefined) : ExpressionNode(Position)
+public record FunctionDeclarationExpressionNode(IdentifierNode[] ParameterNodes, SyntaxNode[] Nodes, bool HasVariableArguments, SourcePosition Position, int LineDefined,SourcePosition EndPosition) : ExpressionNode(Position)
 {
     public override TResult Accept<TContext, TResult>(ISyntaxNodeVisitor<TContext, TResult> visitor, TContext context)
     {

src/Lua/CodeAnalysis/Syntax/Nodes/FunctionDeclarationStatementNode.cs

@@ -1,6 +1,6 @@
 namespace Lua.CodeAnalysis.Syntax.Nodes;
 
-public record FunctionDeclarationStatementNode(ReadOnlyMemory<char> Name, IdentifierNode[] ParameterNodes, SyntaxNode[] Nodes, bool HasVariableArguments, SourcePosition Position,int LineDefined,int LastLineDefined) : StatementNode(Position)
+public record FunctionDeclarationStatementNode(ReadOnlyMemory<char> Name, IdentifierNode[] ParameterNodes, SyntaxNode[] Nodes, bool HasVariableArguments, SourcePosition Position,int LineDefined,SourcePosition EndPosition) : StatementNode(Position)
 {
     public override TResult Accept<TContext, TResult>(ISyntaxNodeVisitor<TContext, TResult> visitor, TContext context)
     {

src/Lua/CodeAnalysis/Syntax/Nodes/IfStatementNode.cs

@@ -4,6 +4,7 @@ public record IfStatementNode(IfStatementNode.ConditionAndThenNodes IfNode, IfSt
 {
     public record ConditionAndThenNodes
     {
+        public SourcePosition Position;
         public required ExpressionNode ConditionNode;
         public required StatementNode[] ThenNodes;
     }

src/Lua/CodeAnalysis/Syntax/Nodes/LocalFunctionDeclarationNode.cs

@@ -1,6 +1,6 @@
 namespace Lua.CodeAnalysis.Syntax.Nodes;
 
-public record LocalFunctionDeclarationStatementNode(ReadOnlyMemory<char> Name, IdentifierNode[] ParameterNodes, SyntaxNode[] Nodes, bool HasVariableArguments, SourcePosition Position, int LineDefined,int LastLineDefined) : FunctionDeclarationStatementNode(Name, ParameterNodes, Nodes, HasVariableArguments, Position, LineDefined, LastLineDefined)
+public record LocalFunctionDeclarationStatementNode(ReadOnlyMemory<char> Name, IdentifierNode[] ParameterNodes, SyntaxNode[] Nodes, bool HasVariableArguments, SourcePosition Position, int LineDefined,SourcePosition EndPosition) : FunctionDeclarationStatementNode(Name, ParameterNodes, Nodes, HasVariableArguments, Position, LineDefined, EndPosition)
 {
     public override TResult Accept<TContext, TResult>(ISyntaxNodeVisitor<TContext, TResult> visitor, TContext context)
     {

src/Lua/CodeAnalysis/Syntax/Nodes/TableMethodDeclarationStatementNode.cs

@@ -1,6 +1,6 @@
 namespace Lua.CodeAnalysis.Syntax.Nodes;
 
-public record TableMethodDeclarationStatementNode(IdentifierNode[] MemberPath, IdentifierNode[] ParameterNodes, StatementNode[] Nodes, bool HasVariableArguments, bool HasSelfParameter, SourcePosition Position, int LineDefined,int LastLineDefined) : StatementNode(Position)
+public record TableMethodDeclarationStatementNode(IdentifierNode[] MemberPath, IdentifierNode[] ParameterNodes, StatementNode[] Nodes, bool HasVariableArguments, bool HasSelfParameter, SourcePosition Position, int LineDefined,SourcePosition EndPosition) : StatementNode(Position)
 {
     public override TResult Accept<TContext, TResult>(ISyntaxNodeVisitor<TContext, TResult> visitor, TContext context)
     {

src/Lua/CodeAnalysis/Syntax/Parser.cs

@@ -33,8 +33,19 @@ public ref struct Parser
             var node = ParseStatement(ref enumerator);
             root.Add(node);
         }
+        var tokensSpan = tokens.AsSpan();
+        var lastToken = tokensSpan[0];
+        for (int i = tokensSpan.Length-1; 0<i;i--)
+        {
+            var t = tokensSpan[i];
+            if (t.Type is not SyntaxTokenType.EndOfLine)
+            {
+                lastToken = t;
+                break;
+            }
+        }
 
-        var tree = new LuaSyntaxTree(root.AsSpan().ToArray());
+        var tree = new LuaSyntaxTree(root.AsSpan().ToArray(),lastToken.Position);
         Dispose();
 
         return tree;
@@ -251,6 +262,7 @@ public ref struct Parser
 
         // skip 'then' keyword
         CheckCurrent(ref enumerator, SyntaxTokenType.Then);
+        var thenToken = enumerator.Current;
 
         using var builder = new PooledList<StatementNode>(64);
         using var elseIfBuilder = new PooledList<IfStatementNode.ConditionAndThenNodes>(64);
@@ -283,6 +295,7 @@ public ref struct Parser
                     case 0:
                         ifNodes = new()
                         {
+                            Position = thenToken.Position,
                             ConditionNode = condition,
                             ThenNodes = builder.AsSpan().ToArray(),
                         };
@@ -291,6 +304,7 @@ public ref struct Parser
                     case 1:
                         elseIfBuilder.Add(new()
                         {
+                            Position = thenToken.Position,
                             ConditionNode = condition,
                             ThenNodes = builder.AsSpan().ToArray(),
                         });
@@ -314,7 +328,7 @@ public ref struct Parser
 
                     // check 'then' keyword
                     CheckCurrent(ref enumerator, SyntaxTokenType.Then);
-
+                    thenToken = enumerator.Current;
                     // set elseif state
                     state = 1;
 
@@ -453,17 +467,17 @@ public ref struct Parser
 
     FunctionDeclarationStatementNode ParseFunctionDeclarationStatement(ref SyntaxTokenEnumerator enumerator, SyntaxToken functionToken)
     {
-        var (Name, Identifiers, Statements, HasVariableArgments, LineDefined, LastLineDefined) = ParseFunctionDeclarationCore(ref enumerator, false);
-        return new FunctionDeclarationStatementNode(Name, Identifiers, Statements, HasVariableArgments, functionToken.Position, LineDefined, LastLineDefined);
+        var (Name, Identifiers, Statements, HasVariableArgments, LineDefined, EndPosition) = ParseFunctionDeclarationCore(ref enumerator, false);
+        return new FunctionDeclarationStatementNode(Name, Identifiers, Statements, HasVariableArgments, functionToken.Position, LineDefined, EndPosition);
     }
 
     LocalFunctionDeclarationStatementNode ParseLocalFunctionDeclarationStatement(ref SyntaxTokenEnumerator enumerator, SyntaxToken functionToken)
     {
-        var (Name, Identifiers, Statements, HasVariableArgments, LineDefined, LastLineDefined) = ParseFunctionDeclarationCore(ref enumerator, false);
-        return new LocalFunctionDeclarationStatementNode(Name, Identifiers, Statements, HasVariableArgments, functionToken.Position, LineDefined, LastLineDefined);
+        var (Name, Identifiers, Statements, HasVariableArgments, LineDefined, EndPosition) = ParseFunctionDeclarationCore(ref enumerator, false);
+        return new LocalFunctionDeclarationStatementNode(Name, Identifiers, Statements, HasVariableArgments, functionToken.Position, LineDefined, EndPosition);
     }
 
-    (ReadOnlyMemory<char> Name, IdentifierNode[] Identifiers, StatementNode[] Statements, bool HasVariableArgments, int LineDefined, int LastLineDefined) ParseFunctionDeclarationCore(ref SyntaxTokenEnumerator enumerator, bool isAnonymous)
+    (ReadOnlyMemory<char> Name, IdentifierNode[] Identifiers, StatementNode[] Statements, bool HasVariableArgments, int LineDefined, SourcePosition EndPosition) ParseFunctionDeclarationCore(ref SyntaxTokenEnumerator enumerator, bool isAnonymous)
     {
         ReadOnlyMemory<char> name;
 
@@ -500,7 +514,7 @@ public ref struct Parser
         // parse statements
         var statements = ParseStatementList(ref enumerator, SyntaxTokenType.End, out var endToken);
 
-        return (name, identifiers, statements, hasVarArg, leftParenToken.Position.Line, endToken.Position.Line);
+        return (name, identifiers, statements, hasVarArg, leftParenToken.Position.Line, endToken.Position);
     }
 
     TableMethodDeclarationStatementNode ParseTableMethodDeclarationStatement(ref SyntaxTokenEnumerator enumerator, SyntaxToken functionToken)
@@ -552,7 +566,7 @@ public ref struct Parser
         // parse statements
         var statements = ParseStatementList(ref enumerator, SyntaxTokenType.End, out var endToken);
 
-        return new TableMethodDeclarationStatementNode(names.AsSpan().ToArray(), identifiers, statements, hasVarArg, hasSelfParameter, functionToken.Position, leftParenToken.Position.Line, endToken.Position.Line);
+        return new TableMethodDeclarationStatementNode(names.AsSpan().ToArray(), identifiers, statements, hasVarArg, hasSelfParameter, functionToken.Position, leftParenToken.Position.Line, endToken.Position);
     }
 
     bool TryParseExpression(ref SyntaxTokenEnumerator enumerator, OperatorPrecedence precedence, [NotNullWhen(true)] out ExpressionNode? result)

tests/Lua.Tests/ParserTests.cs

@@ -17,8 +17,8 @@ else
 end";
             var actual = LuaSyntaxTree.Parse(source).Nodes[0];
             var expected = new IfStatementNode(
-                new() { ConditionNode = new BooleanLiteralNode(true, new(1, 3)), ThenNodes = [] },
-                [new() { ConditionNode = new BooleanLiteralNode(true, new(2, 7)), ThenNodes = [] }],
+                new() { Position = new(1,8),ConditionNode = new BooleanLiteralNode(true, new(1, 3)), ThenNodes = [] },
+                [new() {Position = new(2,13), ConditionNode = new BooleanLiteralNode(true, new(2, 7)), ThenNodes = [] }],
                 [],
                 new(1, 0));