Add a new feature "unimplemented". Tests with that flag will be compiled and run normally, but they will always be counted as part of a set of "unimplemented" counters (failed to compile, successfully compiled, successfully run, failed to run).

Also the digest utility is adjusted to output the unimplemented counters as well (as third part of the "x:y" tuples or in brackets after the single counters). 
The unimplemented count is currently not written to the database.

git-svn-id: branches/svenbarth/misc@19717 -

tests/utils/digest.pp

@@ -22,12 +22,18 @@ uses
 
 const
   failed_to_compile_count : longint = 0;
+  failed_to_compile_unimplemented_count : longint = 0;
   success_compilation_failed_count : longint = 0;
+  success_compilation_failed_unimplemented_count : longint = 0;
   failed_compilation_successful_count : longint = 0;
+  failed_compilation_successful_unimplemented_count : longint = 0;
   successfully_compiled_count : longint = 0;
+  successfully_compiled_unimplemented_count : longint = 0;
   failed_to_run_count : longint = 0;
+  failed_to_run_unimplemented_count : longint = 0;
   known_run_problem : longint = 0;
   successfully_run_count : longint = 0;
+  successfully_run_unimplemented_count : longint = 0;
   skipping_graph_test_count : longint = 0;
   skipping_interactive_test_count : longint = 0;
   skipping_known_bug_count : longint = 0;
@@ -53,7 +59,9 @@ begin
   should_be_run:=next_should_be_run;
   if next_should_be_run and
      (pos(failed_to_run,st)<>1) and
+     (pos(failed_to_run_unimplemented,st)<>1) and
      (pos(successfully_run,st)<>1) and
+     (pos(successfully_run_unimplemented,st)<>1) and
      (pos(skipping_known_bug,st)<>1) and
      (pos(skipping_run_test,st)<>1) and
      (pos(skipping_run_unit,st)<>1) then
@@ -65,19 +73,36 @@ begin
     begin
       inc(failed_to_compile_count);
     end
+  else if pos(failed_to_compile_unimplemented,st)=1 then
+    begin
+      inc(failed_to_compile_unimplemented_count);
+    end
   else if pos(success_compilation_failed,st)=1 then
     begin
       inc(success_compilation_failed_count);
     end
+  else if pos(success_compilation_failed_unimplemented,st)=1 then
+    begin
+      inc(success_compilation_failed_unimplemented_count);
+    end
   else if pos(failed_compilation_successful,st)=1 then
     begin
       inc(failed_compilation_successful_count);
     end
+  else if pos(failed_compilation_successful_unimplemented,st)=1 then
+    begin
+      inc(failed_compilation_successful_unimplemented_count);
+    end
   else if pos(successfully_compiled,st)=1 then
     begin
       inc(successfully_compiled_count);
       next_should_be_run:=true;
     end
+  else if pos(successfully_compiled_unimplemented,st)=1 then
+    begin
+      inc(successfully_compiled_unimplemented_count);
+      next_should_be_run:=true;
+    end
   else if (pos(failed_to_run,st)=1) then
     begin
       inc(failed_to_run_count);
@@ -86,12 +111,28 @@ begin
       if pos(known_problem,st)>0 then
         inc(known_run_problem);
     end
+  else if (pos(failed_to_run_unimplemented,st)=1) then
+    begin
+      inc(failed_to_run_unimplemented_count);
+      { Increase these as well? }
+      {if not should_be_run then
+        inc(unexpected_run);
+      if pos(known_problem,st)>0 then
+        inc(known_run_problem);}
+    end
   else if pos(successfully_run,st)=1 then
     begin
       inc(successfully_run_count);
       if not should_be_run then
         inc(unexpected_run);
     end
+  else if pos(successfully_run_unimplemented,st)=1 then
+    begin
+      inc(successfully_run_unimplemented_count);
+      { Increase this, too? }
+      {if not should_be_run then
+        inc(unexpected_run);}
+    end
   else if pos(skipping_graph_test,st)=1 then
     begin
       inc(skipping_graph_test_count);
@@ -138,10 +179,13 @@ end;
 procedure display_results;
 var
   number_compilations : longint;
+  number_compilations_unimplemented : longint;
   number_skipped : longint;
   number_runs : longint;
+  number_runs_unimplemented : longint;
   all_errors : longint;
   all_success : longint;
+  all_unimplemented : longint;
 
 begin
   all_errors:=failed_to_compile_count
@@ -150,36 +194,62 @@ begin
   all_success:=success_compilation_failed_count
     +successfully_compiled_count
     +successfully_run_count;
+  all_unimplemented:=failed_to_compile_unimplemented_count
+    +failed_compilation_successful_unimplemented_count
+    +failed_to_run_unimplemented_count
+    +success_compilation_failed_unimplemented_count
+    +successfully_compiled_unimplemented_count
+    +successfully_run_unimplemented_count;
   { about compilations }
+  number_compilations_unimplemented:=failed_to_compile_unimplemented_count
+    +success_compilation_failed_unimplemented_count
+    +failed_compilation_successful_unimplemented_count
+    +successfully_compiled_unimplemented_count;
   number_compilations:=failed_to_compile_count
     +success_compilation_failed_count
     +failed_compilation_successful_count
-    +successfully_compiled_count;
+    +successfully_compiled_count
+    +number_compilations_unimplemented;
   { about runs }
-  number_runs:=failed_to_run_count+successfully_run_count;
+  number_runs_unimplemented:=failed_to_run_unimplemented_count
+    +successfully_run_unimplemented_count;
+  number_runs:=failed_to_run_count
+    +successfully_run_count
+    +number_runs_unimplemented;
 
   Writeln('Total = ',number_compilations+number_runs,' (',
     all_errors,':',
-    all_success,')');
+    all_success,':',
+    all_unimplemented,')');
 
   Writeln('Total number of compilations = ', number_compilations,' (',
     failed_to_compile_count+failed_compilation_successful_count,':',
-    successfully_compiled_count+success_compilation_failed_count,')');
-  Writeln('Successfully compiled = ', successfully_compiled_count);
-  Writeln('Successfully failed = ', success_compilation_failed_count);
-  Writeln('Compilation failures = ', failed_to_compile_count);
-  Writeln('Compilation that did not fail while they should = ', failed_compilation_successful_count);
+    successfully_compiled_count+success_compilation_failed_count,':',
+    number_compilations_unimplemented,')');
+  Writeln('Successfully compiled = ', successfully_compiled_count, ' (', successfully_compiled_unimplemented_count, ')');
+  Writeln('Successfully failed = ', success_compilation_failed_count, ' (', success_compilation_failed_unimplemented_count, ')');
+  Writeln('Compilation failures = ', failed_to_compile_count, ' (', failed_to_compile_unimplemented_count, ')');
+  Writeln('Compilation that did not fail while they should = ', failed_compilation_successful_count, ' (', failed_compilation_successful_unimplemented_count, ')');
 
   Writeln('Total number of runs = ', number_runs,' (',
     failed_to_run_count,':',
-    successfully_run_count,')');
-  Writeln('Successful runs = ', successfully_run_count);
-  Writeln('Failed runs = ', failed_to_run_count);
+    successfully_run_count,':',
+    number_runs_unimplemented,')');
+  Writeln('Successful runs = ', successfully_run_count, ' (', successfully_run_unimplemented_count, ')');
+  Writeln('Failed runs = ', failed_to_run_count, ' (', failed_to_run_unimplemented_count, ')');
   if known_run_problem>0 then
     Writeln('From these ',known_run_problem,' known problems');
 
+  Writeln('Number of unimplemented tests = ', all_unimplemented,' (',
+    failed_to_compile_unimplemented_count
+    +failed_compilation_successful_unimplemented_count
+    +failed_to_run_unimplemented_count,':',
+    success_compilation_failed_unimplemented_count
+    +successfully_compiled_unimplemented_count
+    +successfully_run_unimplemented_count,')');
+
   if successfully_compiled_count <>
-     number_runs+skipping_run_unit_count+skipping_run_test_count then
+     number_runs+skipping_run_unit_count+skipping_run_test_count-number_runs_unimplemented then
     begin
       Writeln('Number units compiled = ',skipping_run_unit_count);
       Writeln('Number program that should not be run = ',skipping_run_test_count);

tests/utils/dotest.pp

@@ -681,6 +681,15 @@ begin
 end;
 
 
+function MaybePrependUnimplemented(const LogStr:string):string;
+begin
+  if Config.NotImplemented then
+    MaybePrependUnimplemented:=strunimplemented+LogStr
+  else
+    MaybePrependUnimplemented:=LogStr;
+end;
+
+
 function ExitWithInternalError(const OutName:string):boolean;
 var
   t : text;
@@ -777,13 +786,13 @@ begin
       if (not execres) and (ExecuteResult=0) then
         begin
           AddLog(FailLogFile,TestName);
-          AddLog(ResLogFile,failed_to_compile+PPFileInfo[current]);
+          AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
           AddLog(LongLogFile,line_separation);
-          AddLog(LongLogFile,failed_to_compile+PPFileInfo[current]);
+          AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
           if CopyFile(CompilerLogFile,LongLogFile,true)=0 then
             AddLog(LongLogFile,'IOStatus'+ToStr(IOStatus));
           { avoid to try again }
-          AddLog(ExeLogFile,failed_to_compile+PPFileInfo[current]);
+          AddLog(ExeLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
           Verbose(V_Warning,'IOStatus: '+ToStr(IOStatus));
           exit;
         end;
@@ -794,15 +803,15 @@ begin
          AddLog(FailLogFile,TestName);
          if Config.Note<>'' then
           AddLog(FailLogFile,Config.Note);
-         AddLog(ResLogFile,failed_to_compile+PPFileInfo[current]+' internalerror generated');
+         AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]+' internalerror generated');
          AddLog(LongLogFile,line_separation);
-         AddLog(LongLogFile,failed_to_compile+PPFileInfo[current]);
+         AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
          if Config.Note<>'' then
           AddLog(LongLogFile,Config.Note);
          if CopyFile(CompilerLogFile,LongLogFile,true)=0 then
            AddLog(LongLogFile,'Internal error in compiler');
          { avoid to try again }
-         AddLog(ExeLogFile,failed_to_compile+PPFileInfo[current]);
+         AddLog(ExeLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
          Verbose(V_Warning,'Internal error in compiler');
          exit;
        end;
@@ -813,9 +822,9 @@ begin
    begin
      if ExecuteResult<>0 then
       begin
-        AddLog(ResLogFile,success_compilation_failed+PPFileInfo[current]);
+        AddLog(ResLogFile,MaybePrependUnimplemented(success_compilation_failed)+PPFileInfo[current]);
         { avoid to try again }
-        AddLog(ExeLogFile,success_compilation_failed+PPFileInfo[current]);
+        AddLog(ExeLogFile,MaybePrependUnimplemented(success_compilation_failed)+PPFileInfo[current]);
         RunCompiler:=true;
       end
      else
@@ -823,11 +832,11 @@ begin
         AddLog(FailLogFile,TestName);
         if Config.Note<>'' then
           AddLog(FailLogFile,Config.Note);
-        AddLog(ResLogFile,failed_compilation_successful+PPFileInfo[current]);
+        AddLog(ResLogFile,MaybePrependUnimplemented(failed_compilation_successful)+PPFileInfo[current]);
         AddLog(LongLogFile,line_separation);
-        AddLog(LongLogFile,failed_compilation_successful+PPFileInfo[current]);
+        AddLog(LongLogFile,MaybePrependUnimplemented(failed_compilation_successful)+PPFileInfo[current]);
         { avoid to try again }
-        AddLog(ExeLogFile,failed_compilation_successful+PPFileInfo[current]);
+        AddLog(ExeLogFile,MaybePrependUnimplemented(failed_compilation_successful)+PPFileInfo[current]);
         if Config.Note<>'' then
           AddLog(LongLogFile,Config.Note);
         CopyFile(CompilerLogFile,LongLogFile,true);
@@ -840,10 +849,10 @@ begin
          ((Config.KnownCompileError<>0) and (ExecuteResult=Config.KnownCompileError))) then
       begin
         AddLog(FailLogFile,TestName+known_problem+Config.KnownCompileNote);
-        AddLog(ResLogFile,failed_to_compile+PPFileInfo[current]+known_problem+Config.KnownCompileNote);
+        AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]+known_problem+Config.KnownCompileNote);
         AddLog(LongLogFile,line_separation);
         AddLog(LongLogFile,known_problem+Config.KnownCompileNote);
-        AddLog(LongLogFile,failed_to_compile+PPFileInfo[current]+' ('+ToStr(ExecuteResult)+')');
+        AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]+' ('+ToStr(ExecuteResult)+')');
         if Copyfile(CompilerLogFile,LongLogFile,true)=0 then
           AddLog(LongLogFile,known_problem+'exitcode: '+ToStr(ExecuteResult));
         Verbose(V_Warning,known_problem+'exitcode: '+ToStr(ExecuteResult));
@@ -853,20 +862,20 @@ begin
         AddLog(FailLogFile,TestName);
         if Config.Note<>'' then
           AddLog(FailLogFile,Config.Note);
-        AddLog(ResLogFile,failed_to_compile+PPFileInfo[current]);
+        AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
         AddLog(LongLogFile,line_separation);
-        AddLog(LongLogFile,failed_to_compile+PPFileInfo[current]);
+        AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
         if Config.Note<>'' then
           AddLog(LongLogFile,Config.Note);
         if CopyFile(CompilerLogFile,LongLogFile,true)=0 then
           AddLog(LongLogFile,'Exitcode: '+ToStr(ExecuteResult)+' (expected 0)');
         { avoid to try again }
-        AddLog(ExeLogFile,failed_to_compile+PPFileInfo[current]);
+        AddLog(ExeLogFile,MaybePrependUnimplemented(failed_to_compile)+PPFileInfo[current]);
         Verbose(V_Warning,'Exitcode: '+ToStr(ExecuteResult)+' (expected 0)');
       end
      else
       begin
-        AddLog(ResLogFile,successfully_compiled+PPFileInfo[current]);
+        AddLog(ResLogFile,MaybePrependUnimplemented(successfully_compiled)+PPFileInfo[current]);
         RunCompiler:=true;
       end;
    end;
@@ -1174,13 +1183,13 @@ begin
   if (not execres) and (ExecuteResult=0) then
     begin
       AddLog(FailLogFile,TestName);
-      AddLog(ResLogFile,failed_to_run+PPFileInfo[current]);
+      AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]);
       AddLog(LongLogFile,line_separation);
-      AddLog(LongLogFile,failed_to_run+PPFileInfo[current]);
+      AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]);
       if CopyFile(EXELogFile,LongLogFile,true)=0 then
         AddLog(LongLogFile,'IOStatus: '+ToStr(IOStatus));
       { avoid to try again }
-      AddLog(ExeLogFile,failed_to_run+PPFileInfo[current]);
+      AddLog(ExeLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]);
       Verbose(V_Warning,'IOStatus: '+ToStr(IOStatus));
       exit;
     end;
@@ -1191,10 +1200,10 @@ begin
         (ExecuteResult=Config.KnownRunError) then
        begin
          AddLog(FailLogFile,TestName+known_problem+Config.KnownRunNote);
-         AddLog(ResLogFile,failed_to_run+PPFileInfo[current]+known_problem+Config.KnownRunNote);
+         AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]+known_problem+Config.KnownRunNote);
          AddLog(LongLogFile,line_separation);
          AddLog(LongLogFile,known_problem+Config.KnownRunNote);
-         AddLog(LongLogFile,failed_to_run+PPFileInfo[current]+' ('+ToStr(ExecuteResult)+')');
+         AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]+' ('+ToStr(ExecuteResult)+')');
          if Copyfile(EXELogFile,LongLogFile,true)=0 then
            begin
              AddLog(LongLogFile,known_problem+'exitcode: '+ToStr(ExecuteResult)+' (expected '+ToStr(Config.ResultCode)+')');
@@ -1205,9 +1214,9 @@ begin
      else
        begin
          AddLog(FailLogFile,TestName);
-         AddLog(ResLogFile,failed_to_run+PPFileInfo[current]);
+         AddLog(ResLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]);
          AddLog(LongLogFile,line_separation);
-         AddLog(LongLogFile,failed_to_run+PPFileInfo[current]+' ('+ToStr(ExecuteResult)+')');
+         AddLog(LongLogFile,MaybePrependUnimplemented(failed_to_run)+PPFileInfo[current]+' ('+ToStr(ExecuteResult)+')');
          if Copyfile(EXELogFile,LongLogFile,true)=0 then
            begin
              AddLog(LongLogFile,'Exitcode: '+ToStr(ExecuteResult)+' (expected '+ToStr(Config.ResultCode)+')');
@@ -1218,7 +1227,7 @@ begin
    end
   else
    begin
-     AddLog(ResLogFile,successfully_run+PPFileInfo[current]);
+     AddLog(ResLogFile,MaybePrependUnimplemented(successfully_run)+PPFileInfo[current]);
      RunExecutable:=true;
    end;
 

tests/utils/teststr.pp

@@ -20,12 +20,19 @@ unit teststr;
 interface
 
 const
+  strunimplemented = '(unimplemented) ';
   failed_to_compile = 'Failed to compile ';
+  failed_to_compile_unimplemented = strunimplemented + failed_to_compile;
   success_compilation_failed = 'Success, compilation failed ';
+  success_compilation_failed_unimplemented = strunimplemented + success_compilation_failed;
   failed_compilation_successful = 'Failed, compilation successful ';
+  failed_compilation_successful_unimplemented = strunimplemented + failed_compilation_successful;
   successfully_compiled = 'Successfully compiled ';
+  successfully_compiled_unimplemented = strunimplemented + successfully_compiled;
   failed_to_run = 'Failed to run ';
+  failed_to_run_unimplemented = strunimplemented + failed_to_run;
   successfully_run = 'Successfully run ';
+  successfully_run_unimplemented = strunimplemented + successfully_run;
   skipping_graph_test = 'Skipping test because it uses graph ';
   skipping_interactive_test = 'Skipping test because it is interactive ';
   skipping_known_bug = 'Skipping test because it is a known bug ';

tests/utils/testu.pp

@@ -41,6 +41,7 @@ type
     Files         : string;
     WpoParas      : string;
     WpoPasses     : longint;
+    NotImplemented: boolean;
   end;
 
 Const
@@ -275,6 +276,9 @@ begin
               else
                 if GetEntry('WPOPASSES') then
                  val(res,r.wpopasses,code)
+              else
+                if GetEntry('UNIMPLEMENTED') then
+                 r.NotImplemented:=true
               else
                Verbose(V_Error,'Unknown entry: '+s);
             end;