Avoid blocking pipe read calls by only asking as many bytes as returned by NumBytesAvailable

git-svn-id: trunk@29303 -

packages/fcl-process/src/process.pp

@@ -180,6 +180,7 @@ function RunCommandInDir(const curdir,cmdline:string;var outputstring:string):bo
 function RunCommand(const exename:string;const commands:array of string;var outputstring:string):boolean;
 function RunCommand(const cmdline:string;var outputstring:string):boolean; deprecated;
 
+
 implementation
 
 {$i process.inc}
@@ -472,33 +473,88 @@ Const
   READ_BYTES = 65536; // not too small to avoid fragmentation when reading large files.
 
 // helperfunction that does the bulk of the work.
-function internalRuncommand(p:TProcess;var outputstring:string;var exitstatus:integer):integer;
+// We need to also collect stderr output in order to avoid
+// lock out if the stderr pipe is full.
+function internalRuncommand(p:TProcess;var outputstring:string;
+                            var stderrstring:string; var exitstatus:integer):integer;
 var
-    numbytes,bytesread : integer;
+    numbytes,bytesread,available : integer;
+    outputlength, stderrlength : integer;
+    stderrnumbytes,stderrbytesread : integer;
 begin
   result:=-1;
   try
     try
     p.Options :=  [poUsePipes];
     bytesread:=0;
+    outputlength:=0;
+    stderrbytesread:=0;
+    stderrlength:=0;
     p.Execute;
     while p.Running do
       begin
-        Setlength(outputstring,BytesRead + READ_BYTES);
-        NumBytes := p.Output.Read(outputstring[1+bytesread], READ_BYTES);
-        if NumBytes > 0 then
-          Inc(BytesRead, NumBytes)
+        // Only call ReadFromStream if Data from corresponding stream
+        // is already available, otherwise, on  linux, the read call
+        // is blocking, and thus it is not possible to be sure to handle
+        // big data amounts bboth on output and stderr pipes. PM.
+        available:=P.Output.NumBytesAvailable;
+        if  available > 0 then
+          begin
+            if (BytesRead + available > outputlength) then
+              begin
+                outputlength:=BytesRead + READ_BYTES;
+                Setlength(outputstring,outputlength);
+              end;
+            NumBytes := p.Output.Read(outputstring[1+bytesread], available);
+            if NumBytes > 0 then
+              Inc(BytesRead, NumBytes);
+          end
+        // The check for assigned(P.stderr) is mainly here so that
+        // if we use poStderrToOutput in p.Options, we do not access invalid memory.
+        else if assigned(P.stderr) and (P.StdErr.NumBytesAvailable > 0) then
+          begin
+            available:=P.StdErr.NumBytesAvailable;
+            if (StderrBytesRead + available > stderrlength) then
+              begin
+                stderrlength:=StderrBytesRead + READ_BYTES;
+                Setlength(stderrstring,stderrlength);
+              end;
+            StderrNumBytes := p.StdErr.Read(stderrstring[1+StderrBytesRead], available);
+            if StderrNumBytes > 0 then
+              Inc(StderrBytesRead, StderrNumBytes);
+          end
         else
           Sleep(100);
       end;
-    repeat
-      Setlength(outputstring,BytesRead + READ_BYTES);
-      NumBytes := p.Output.Read(outputstring[1+bytesread], READ_BYTES);
-      if NumBytes > 0 then
-        Inc(BytesRead, NumBytes);
-    until NumBytes <= 0;
+    // Get left output after end of execution
+    available:=P.Output.NumBytesAvailable;
+    while available > 0 do
+      begin
+        if (BytesRead + available > outputlength) then
+          begin
+            outputlength:=BytesRead + READ_BYTES;
+            Setlength(outputstring,outputlength);
+          end;
+        NumBytes := p.Output.Read(outputstring[1+bytesread], available);
+        if NumBytes > 0 then
+          Inc(BytesRead, NumBytes);
+        available:=P.Output.NumBytesAvailable;
+      end;
     setlength(outputstring,BytesRead);
-    exitstatus:=p.exitstatus;	
+    while assigned(P.stderr) and (P.Stderr.NumBytesAvailable > 0) do
+      begin
+        available:=P.Stderr.NumBytesAvailable;
+        if (StderrBytesRead + available > stderrlength) then
+          begin
+            stderrlength:=StderrBytesRead + READ_BYTES;
+            Setlength(stderrstring,stderrlength);
+          end;
+        StderrNumBytes := p.StdErr.Read(stderrstring[1+StderrBytesRead], available);
+        if StderrNumBytes > 0 then
+          Inc(StderrBytesRead, StderrNumBytes);
+      end;
+    setlength(stderrstring,StderrBytesRead);
+    exitstatus:=p.exitstatus;
     result:=0; // we came to here, document that.
     except
       on e : Exception do
@@ -512,10 +568,13 @@ begin
     end;
 end;
 
+{ Functions without StderrString }
+
 function RunCommandIndir(const curdir:string;const exename:string;const commands:array of string;var outputstring:string;var exitstatus:integer):integer;
 Var
     p : TProcess;
     i : integer;
+    ErrorString : String;
 begin
   p:=TProcess.create(nil);
   p.Executable:=exename;
@@ -524,19 +583,20 @@ begin
   if high(commands)>=0 then
    for i:=low(commands) to high(commands) do
      p.Parameters.add(commands[i]);
-  result:=internalruncommand(p,outputstring,exitstatus);
+  result:=internalruncommand(p,outputstring,errorstring,exitstatus);
 end;
 
 function RunCommandInDir(const curdir,cmdline:string;var outputstring:string):boolean; deprecated;
 Var
     p : TProcess;
     exitstatus : integer;
+    ErrorString : String;
 begin
   p:=TProcess.create(nil);
   p.setcommandline(cmdline);
   if curdir<>'' then
     p.CurrentDirectory:=curdir;
-  result:=internalruncommand(p,outputstring,exitstatus)=0;
+  result:=internalruncommand(p,outputstring,errorstring,exitstatus)=0;
   if exitstatus<>0 then result:=false;
 end;
 
@@ -545,6 +605,7 @@ Var
     p : TProcess;
     i,
     exitstatus : integer;
+    ErrorString : String;
 begin
   p:=TProcess.create(nil);
   p.Executable:=exename;
@@ -553,7 +614,7 @@ begin
   if high(commands)>=0 then
    for i:=low(commands) to high(commands) do
      p.Parameters.add(commands[i]);
-  result:=internalruncommand(p,outputstring,exitstatus)=0;
+  result:=internalruncommand(p,outputstring,errorstring,exitstatus)=0;
   if exitstatus<>0 then result:=false;
 end;
 
@@ -561,10 +622,11 @@ function RunCommand(const cmdline:string;var outputstring:string):boolean; depre
 Var
     p : TProcess;
     exitstatus : integer;
+    ErrorString : String;
 begin
   p:=TProcess.create(nil);
   p.setcommandline(cmdline);
-  result:=internalruncommand(p,outputstring,exitstatus)=0;
+  result:=internalruncommand(p,outputstring,errorstring,exitstatus)=0;
   if exitstatus<>0 then result:=false;
 end;
 
@@ -573,13 +635,14 @@ Var
     p : TProcess;
     i,
     exitstatus : integer;
+    ErrorString : String;
 begin
   p:=TProcess.create(nil);
   p.Executable:=exename;
   if high(commands)>=0 then
    for i:=low(commands) to high(commands) do
      p.Parameters.add(commands[i]);
-  result:=internalruncommand(p,outputstring,exitstatus)=0;
+  result:=internalruncommand(p,outputstring,errorstring,exitstatus)=0;
   if exitstatus<>0 then result:=false;
 end;
 

packages/fpmkunit/src/fpmkunit.pp

@@ -1721,7 +1721,7 @@ var
     //ifdef the MsgNum so it contains the correct message numbers for each compiler version.
     MsgNum : array [TMessages] of integer = (3104, 9015);
 
-    n: longint;
+    n,available: longint;
     BuffPos: longint;
     sLine: string;
     ch: char;
@@ -1729,14 +1729,24 @@ var
     ipos: integer;
     snum: string;
   begin
-    // make sure we have room
-    ConsoleOutput.SetSize(BytesRead + READ_BYTES);
 
     // try reading it
     if ReadFromStdErr then
-      n := P.Stderr.Read((ConsoleOutput.Memory + BytesRead)^, READ_BYTES)
+      begin
+        available:=P.Stderr.NumBytesAvailable;
+        // make sure we have room
+        if (bytesRead + Available > ConsoleOutput.Size) then
+          ConsoleOutput.SetSize(BytesRead + Available);
+        n := P.Stderr.Read((ConsoleOutput.Memory + BytesRead)^, available);
+      end
     else
-      n := P.Output.Read((ConsoleOutput.Memory + BytesRead)^, READ_BYTES);
+      begin
+        available:=P.Output.NumBytesAvailable;
+        // make sure we have room
+        if (bytesRead + Available > ConsoleOutput.Size) then
+          ConsoleOutput.SetSize(BytesRead + Available);
+        n := P.Output.Read((ConsoleOutput.Memory + BytesRead)^, available);
+      end;
     if n > 0 then
     begin
       Inc(BytesRead, n);