* various fixes for fully setting/restoring the token position, seems to

    fix the random garbage before the initial "const" of msgidx.inc in
    verbose.s when compiling with -al

git-svn-id: trunk@13578 -

compiler/scanner.pas

@@ -70,6 +70,12 @@ interface
        tspecialgenerictoken = (ST_LOADSETTINGS,ST_LINE,ST_COLUMN,ST_FILEINDEX);
 
        tscannerfile = class
+       private
+         procedure do_gettokenpos(out tokenpos: longint; out filepos: tfileposinfo);
+         procedure cachenexttokenpos;
+         procedure setnexttoken;
+         procedure savetokenpos;
+         procedure restoretokenpos;
        public
           inputfile    : tinputfile;  { current inputfile list }
           inputfilecount : longint;
@@ -81,10 +87,16 @@ interface
           line_no,                    { line }
           lastlinepos  : longint;
 
-          lasttokenpos : longint;     { token }
+          lasttokenpos,
+          nexttokenpos : longint;     { token }
           lasttoken,
           nexttoken    : ttoken;
 
+          oldlasttokenpos     : longint; { temporary saving/restoring tokenpos }
+          oldcurrent_filepos,
+          oldcurrent_tokenpos : tfileposinfo;
+
+
           replaysavetoken : ttoken;
           replaytokenbuf,
           recordtokenbuf : tdynamicarray;
@@ -95,7 +107,9 @@ interface
           last_settings : tsettings;
 
           { last filepos we stored }
-          last_filepos : tfileposinfo;
+          last_filepos,
+          { if nexttoken<>NOTOKEN, then nexttokenpos holds its filepos }
+          next_filepos   : tfileposinfo;
 
           comment_level,
           yylexcount     : longint;
@@ -1828,6 +1842,7 @@ In case not, the value returned can be arbitrary.
         line_no:=0;
         lastlinepos:=0;
         lasttokenpos:=0;
+        nexttokenpos:=0;
         lasttoken:=NOTOKEN;
         nexttoken:=NOTOKEN;
         lastasmgetchar:=#0;
@@ -1873,6 +1888,7 @@ In case not, the value returned can be arbitrary.
         line_no:=0;
         lastlinepos:=0;
         lasttokenpos:=0;
+        nexttokenpos:=0;
       end;
 
 
@@ -1887,6 +1903,7 @@ In case not, the value returned can be arbitrary.
         line_no:=0;
         lastlinepos:=0;
         lasttokenpos:=0;
+        nexttokenpos:=0;
       end;
 
 
@@ -2212,7 +2229,7 @@ In case not, the value returned can be arbitrary.
 
                    line_no:=1;
                    if cs_asm_source in current_settings.globalswitches then
-                     inputfile.setline(line_no,bufstart);
+                     inputfile.setline(line_no,inputstart+inputpointer-inputbuffer);
                  end;
               end
              else
@@ -2265,27 +2282,65 @@ In case not, the value returned can be arbitrary.
         line_no:=line;
         lastlinepos:=0;
         lasttokenpos:=0;
+        nexttokenpos:=0;
       { load new c }
         c:=inputpointer^;
         inc(inputpointer);
       end;
 
 
+    procedure tscannerfile.do_gettokenpos(out tokenpos: longint; out filepos: tfileposinfo);
+      begin
+        tokenpos:=inputstart+(inputpointer-inputbuffer);
+        filepos.line:=line_no;
+        filepos.column:=tokenpos-lastlinepos;
+        filepos.fileindex:=inputfile.ref_index;
+        filepos.moduleindex:=current_module.unit_index;
+      end;
+
+
     procedure tscannerfile.gettokenpos;
     { load the values of tokenpos and lasttokenpos }
       begin
-        lasttokenpos:=inputstart+(inputpointer-inputbuffer);
-        current_tokenpos.line:=line_no;
-        current_tokenpos.column:=lasttokenpos-lastlinepos;
-        current_tokenpos.fileindex:=inputfile.ref_index;
-        current_tokenpos.moduleindex:=current_module.unit_index;
+        do_gettokenpos(lasttokenpos,current_tokenpos);
+        current_filepos:=current_tokenpos;
+      end;
+
+
+    procedure tscannerfile.cachenexttokenpos;
+      begin
+        do_gettokenpos(nexttokenpos,next_filepos);
+      end;
+
+
+    procedure tscannerfile.setnexttoken;
+      begin
+        token:=nexttoken;
+        nexttoken:=NOTOKEN;
+        lasttokenpos:=nexttokenpos;
+        current_tokenpos:=next_filepos;
         current_filepos:=current_tokenpos;
+        nexttokenpos:=0;
+      end;
+
+
+    procedure tscannerfile.savetokenpos;
+      begin
+        oldlasttokenpos:=lasttokenpos;
+        oldcurrent_filepos:=current_filepos;
+        oldcurrent_tokenpos:=current_tokenpos;
+      end;
+
+
+    procedure tscannerfile.restoretokenpos;
+      begin
+        lasttokenpos:=oldlasttokenpos;
+        current_filepos:=oldcurrent_filepos;
+        current_tokenpos:=oldcurrent_tokenpos;
       end;
 
 
     procedure tscannerfile.inc_comment_level;
-      var
-         oldcurrent_filepos : tfileposinfo;
       begin
          if (m_nested_comment in current_settings.modeswitches) then
            inc(comment_level)
@@ -2293,10 +2348,10 @@ In case not, the value returned can be arbitrary.
            comment_level:=1;
          if (comment_level>1) then
           begin
-             oldcurrent_filepos:=current_filepos;
+             savetokenpos;
              gettokenpos; { update for warning }
              Message1(scan_w_comment_level,tostr(comment_level));
-             current_filepos:=oldcurrent_filepos;
+             restoretokenpos;
           end;
       end;
 
@@ -2313,8 +2368,6 @@ In case not, the value returned can be arbitrary.
     procedure tscannerfile.linebreak;
       var
          cur : char;
-         oldtokenpos,
-         oldcurrent_filepos : tfileposinfo;
       begin
         with inputfile do
          begin
@@ -2334,20 +2387,18 @@ In case not, the value returned can be arbitrary.
            { Always return #10 as line break }
            c:=#10;
            { increase line counters }
-           lastlinepos:=bufstart+(inputpointer-inputbuffer);
+           lastlinepos:=inputstart+(inputpointer-inputbuffer);
            inc(line_no);
            { update linebuffer }
            if cs_asm_source in current_settings.globalswitches then
              inputfile.setline(line_no,lastlinepos);
            { update for status and call the show status routine,
              but don't touch current_filepos ! }
-           oldcurrent_filepos:=current_filepos;
-           oldtokenpos:=current_tokenpos;
+           savetokenpos;
            gettokenpos; { update for v_status }
            inc(status.compiledlines);
            ShowStatus;
-           current_filepos:=oldcurrent_filepos;
-           current_tokenpos:=oldtokenpos;
+           restoretokenpos;
          end;
       end;
 
@@ -2482,10 +2533,8 @@ In case not, the value returned can be arbitrary.
 
 
     procedure tscannerfile.handleconditional(p:tdirectiveitem);
-      var
-        oldcurrent_filepos : tfileposinfo;
       begin
-        oldcurrent_filepos:=current_filepos;
+        savetokenpos;
         repeat
           current_scanner.gettokenpos;
           p.proc();
@@ -2507,7 +2556,7 @@ In case not, the value returned can be arbitrary.
              Message1(scan_d_handling_switch,'$'+p.name);
            end;
         until false;
-        current_filepos:=oldcurrent_filepos;
+        restoretokenpos;
       end;
 
 
@@ -3226,8 +3275,7 @@ In case not, the value returned can be arbitrary.
       { was there already a token read, then return that token }
         if nexttoken<>NOTOKEN then
          begin
-           token:=nexttoken;
-           nexttoken:=NOTOKEN;
+           setnexttoken;
            goto exit_label;
          end;
 
@@ -3376,6 +3424,7 @@ In case not, the value returned can be arbitrary.
                   { first check for a . }
                     if c='.' then
                      begin
+                       cachenexttokenpos;
                        readchar;
                        { is it a .. from a range? }
                        case c of