Initial code for setup.py helper class

Uses helper thread to automatically keep track of
subprocess.Popen output

go/setup.py

@@ -3,19 +3,113 @@ import sys
 import os
 import setup_util
 
+
+from threading import Thread
+from Queue import Queue, Empty
+
+# TODO - no need to use a daemon, kill this off in stop!
+# NOTE - it is safe to use logging module in a multi-threaded
+# system, but not safe to log to the same file across multiple 
+# processes. Our system has two main processes (main and __run_test), 
+# and lots of minor ones from 'subprocess'. As long as we only use
+# one logger inside TestRunner and NonBlockingFoo, sd are good
+# Add credit for http://eyalarubas.com/python-subproc-nonblock.html
+class NonBlockingStreamReader:
+
+  def __init__(self, stream):
+    self._s = stream
+    self._q = Queue()
+
+    def _populateQueue(stream, queue):
+      for line in iter(stream.readline, b''):
+        queue.put(line)
+
+    self._t = Thread(target = _populateQueue,
+                args = (self._s, self._q))
+    self._t.daemon = True
+    self._t.start() #start collecting lines from the stream
+
+  # TODO  - This is only returning one line, if it is available. 
+  def readline(self, timeout = None):
+    try:
+      return self._q.get(block = timeout is not None,
+                 timeout = timeout)
+    except Empty:
+      return None
+  
+  def read(self):
+    lines = []
+    while True:
+      line = self.readline(0.1)
+      if not line:
+        return lines
+      lines.append(line)
+
+import time
+
+class TestRunner:
+  def __init__(self, directory, stdout):
+    self.dir = directory
+    self.stdout = stdout
+
+  def sh(self, command, **kwargs):
+    kwargs.setdefault('cwd', self.dir)
+    self.stdout.write("Running %s (cwd=%s)" % (command, kwargs.get('cwd')))
+    try:
+      output = subprocess.check_output(command, shell=True, **kwargs)
+      if output and output.strip():
+        self.stdout.write("Output:")
+        self.stdout.write(output.rstrip('\n'))
+      else:
+        self.stdout.write("No Output")
+    except subprocess.CalledProcessError:
+      self.stdout.write("Process Returned non-zero exit code")
+
+  def sh_async(self, command, initial_logs=True, **kwargs):
+    kwargs.setdefault('cwd', self.dir)
+    self.stdout.write("Running %s (cwd=%s)" % (command, kwargs.get('cwd')))
+    # Open in line-buffered mode, as NonBlockingStreamReader uses readline anyways
+    process = subprocess.Popen(command, bufsize=1, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, **kwargs)
+    nbsr = NonBlockingStreamReader(process.stdout)
+    nbsr_err = NonBlockingStreamReader(process.stderr) 
+    if initial_logs:
+      time.sleep(8)
+      # TODO put this read into a tight loop to prevent deadlock due to 
+      # filling up OS buffers
+      out = nbsr.read()
+      err = nbsr_err.read()
+      if len(out) != 0:
+        self.stdout.write("Initial Output:")
+	for line in out:
+          self.stdout.write(line.rstrip('\n'))
+      else:
+        self.stdout.write("No output")
+      if len(err) != 0:
+        self.stdout.write("Initial Error Logs")
+	for line in err:
+          self.stdout.write(line.rstrip('\n'))
+
 def start(args, logfile, errfile):
+  t = TestRunner("go", logfile)
   setup_util.replace_text("go/src/hello/hello.go", "tcp\(.*:3306\)", "tcp(" + args.database_host + ":3306)")
   if os.name == 'nt':
     #subprocess.call("rmdir /s /q pkg\\windows_amd64", shell=True, cwd="go")
     #subprocess.call("rmdir /s /q src\\github.com", shell=True, cwd="go")
     #subprocess.call("del /s /q /f bin\\hello.exe", shell=True, cwd="go")
     subprocess.call("set GOPATH=C:\\FrameworkBenchmarks\\go&& go get ./...", shell=True, cwd="go", stderr=errfile, stdout=logfile)
-    subprocess.Popen("setup.bat", shell=True, cwd="go", stderr=errfile, stdout=logfile) 
+    subprocess.Popen("setup.bat", shell=True, cwd="go", stderr=errfile, stdout=logfile)
+
     return 0
-  # os.environ["GOPATH"] = os.path.expanduser('~/FrameworkBenchmarks/go')
-  subprocess.call("go get ./...", shell=True, cwd="go", stderr=errfile, stdout=logfile) 
-  subprocess.Popen("go run src/hello/hello.go".rsplit(" "), cwd="go", stderr=errfile, stdout=logfile)
+  
+  t.sh("which go")
+  t.sh("rm -rf src/github.com")
+  t.sh("ls src/github.com/go-sql-driver/mysql")
+  t.sh("go get ./...")
+  t.sh("ls src/github.com/go-sql-driver/mysql")
+  t.sh_async("go run -x -v src/hello/hello.go")
+  # t.sh("ps aux")
   return 0
+
 def stop(logfile, errfile):
   if os.name == 'nt':
     subprocess.call("taskkill /f /im go.exe > NUL", shell=True, stderr=errfile, stdout=logfile)