Merge https://github.com/TechEmpower/FrameworkBenchmarks into development

HttpListener/setup.ps1

@@ -1,14 +1,32 @@
 param($action)
 
+$ErrorActionPreference = 'Stop'
+
+# From http://zduck.com/2012/powershell-batch-files-exit-codes/
+function Exec
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Position=0, Mandatory=1)]
+        [scriptblock]$Command,
+        [Parameter(Position=1, Mandatory=0)]
+        [string]$ErrorMessage = "Execution of command failed.`n$Command"
+    )
+    & $Command
+    if ($LastExitCode -ne 0) {
+        throw "Exec: $ErrorMessage"
+    }
+}
+
 $root = "C:\FrameworkBenchmarks\HttpListener"
 $msbuild = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe"
 
 # Stop
-Stop-Process -Name HttpListener -ErrorAction SilentlyContinue
+Get-Process | Where-Object { $_.Name -ieq "HttpListener" } | Stop-Process
 
 if ($action -eq 'start') {
     # Build the project
-    &$msbuild "$root\HttpListener.sln" /p:DownloadNuGetExe=true /p:RequireRestoreConsent=false /p:Configuration=Release /t:Rebuild
+    Exec { & $msbuild "$root\HttpListener.sln" /p:DownloadNuGetExe=true /p:RequireRestoreConsent=false /p:Configuration=Release /t:Rebuild }
     
     Start-Process "$root\HttpListener\bin\Release\HttpListener.exe"
 }

HttpListener/setup.py

@@ -9,7 +9,7 @@ def start(args):
   
   try:
     setup_util.replace_text("HttpListener/HttpListener/App.config", "localhost", args.database_host)
-    subprocess.check_call("powershell -File setup.ps1 start", cwd="HttpListener")
+    subprocess.check_call("powershell -Command .\\setup.ps1 start", cwd="HttpListener")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,5 +18,5 @@ def stop():
   if os.name != 'nt':
     return 0
   
-  subprocess.Popen("powershell -File setup.ps1 stop", cwd="HttpListener")
+  subprocess.check_call("powershell -Command .\\setup.ps1 stop", cwd="HttpListener")
   return 0

aspnet-stripped/setup_iis.ps1

@@ -1,5 +1,23 @@
 param($action)
 
+$ErrorActionPreference = 'Stop'
+
+# From http://zduck.com/2012/powershell-batch-files-exit-codes/
+function Exec
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Position=0, Mandatory=1)]
+        [scriptblock]$Command,
+        [Parameter(Position=1, Mandatory=0)]
+        [string]$ErrorMessage = "Execution of command failed.`n$Command"
+    )
+    & $Command
+    if ($LastExitCode -ne 0) {
+        throw "Exec: $ErrorMessage"
+    }
+}
+
 $source = "C:\FrameworkBenchmarks\aspnet-stripped\src"
 
 # Stop
@@ -8,7 +26,7 @@ if (Get-WebSite -Name Benchmarks) { Remove-WebSite -Name Benchmarks }
 if ($action -eq 'start') {
     # Because we don't use msbuild to compile the code, we do this all manually.
     
-    & .\NuGet.exe install -o src\packages src\packages.config
+    Exec { & .\NuGet.exe install -o src\packages src\packages.config }
 
     if (-Not (Test-Path src\bin)) { New-Item -Path src\bin -ItemType directory | Out-Null }
 

aspnet-stripped/setup_iis.py

@@ -9,7 +9,7 @@ def start(args):
   
   try:
     setup_util.replace_text("aspnet-stripped/src/Web.config", "localhost", args.database_host)
-    subprocess.check_call("powershell -File setup_iis.ps1 start", cwd="aspnet-stripped")
+    subprocess.check_call("powershell -Command .\\setup_iis.ps1 start", cwd="aspnet-stripped")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,5 +18,5 @@ def stop():
   if os.name != 'nt':
     return 0
   
-  subprocess.Popen("powershell -File setup_iis.ps1 stop", cwd="aspnet-stripped")
+  subprocess.check_call("powershell -Command .\\setup_iis.ps1 stop", cwd="aspnet-stripped")
   return 0

aspnet/setup_iis.ps1

@@ -1,5 +1,23 @@
 param($action)
 
+$ErrorActionPreference = 'Stop'
+
+# From http://zduck.com/2012/powershell-batch-files-exit-codes/
+function Exec
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Position=0, Mandatory=1)]
+        [scriptblock]$Command,
+        [Parameter(Position=1, Mandatory=0)]
+        [string]$ErrorMessage = "Execution of command failed.`n$Command"
+    )
+    & $Command
+    if ($LastExitCode -ne 0) {
+        throw "Exec: $ErrorMessage"
+    }
+}
+
 $wwwroot = "C:\FrameworkBenchmarks\aspnet\www"
 $source = "C:\FrameworkBenchmarks\aspnet\src"
 $msbuild = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe"
@@ -15,7 +33,7 @@ if ($action -eq 'start') {
     New-WebSite -Name Benchmarks -Port 8080 -PhysicalPath $wwwroot
     
     # Build the project
-    &$msbuild "$source\Benchmarks.AspNet.csproj" /t:RestorePackages
-    &$msbuild "$source\Benchmarks.AspNet.csproj" /p:Configuration=Release /p:Platform=x64 /t:Clean
-    &$msbuild "$source\Benchmarks.AspNet.csproj" /p:Configuration=Release /p:Platform=x64 /p:DeployOnBuild=true /p:PublishProfile=IIS
+    Exec { & $msbuild "$source\Benchmarks.AspNet.csproj" /t:RestorePackages }
+    Exec { & $msbuild "$source\Benchmarks.AspNet.csproj" /p:Configuration=Release /p:Platform=x64 /t:Clean }
+    Exec { & $msbuild "$source\Benchmarks.AspNet.csproj" /p:Configuration=Release /p:Platform=x64 /p:DeployOnBuild=true /p:PublishProfile=IIS }
 }

aspnet/setup_iis.py

@@ -9,7 +9,7 @@ def start(args):
   
   try:
     setup_util.replace_text("aspnet/src/Web.config", "localhost", args.database_host)
-    subprocess.check_call("powershell -File setup_iis.ps1 start", cwd="aspnet")
+    subprocess.check_call("powershell -Command .\\setup_iis.ps1 start", cwd="aspnet")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,5 +18,5 @@ def stop():
   if os.name != 'nt':
     return 0
   
-  subprocess.Popen("powershell -File setup_iis.ps1 stop", cwd="aspnet")
+  subprocess.check_call("powershell -Command .\\setup_iis.ps1 stop", cwd="aspnet")
   return 0

nancy/setup_iis.ps1

@@ -1,7 +1,26 @@
 param($action)
 
+$ErrorActionPreference = 'Stop'
+
+# From http://zduck.com/2012/powershell-batch-files-exit-codes/
+function Exec
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Position=0, Mandatory=1)]
+        [scriptblock]$Command,
+        [Parameter(Position=1, Mandatory=0)]
+        [string]$ErrorMessage = "Execution of command failed.`n$Command"
+    )
+    & $Command
+    if ($LastExitCode -ne 0) {
+        throw "Exec: $ErrorMessage"
+    }
+}
+
 $wwwroot = "C:\FrameworkBenchmarks\nancy\www"
 $source = "C:\FrameworkBenchmarks\nancy\src"
+$msbuild = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe"
 
 # Stop
 if (Get-WebSite -Name Benchmarks) { Remove-WebSite -Name Benchmarks }
@@ -14,6 +33,6 @@ if ($action -eq 'start') {
     New-WebSite -Name Benchmarks -Port 8080 -PhysicalPath $wwwroot
     
     # Build the project
-    C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe "$source\NancyBenchmark.csproj" /p:Configuration=Release /p:Platform="AnyCPU" /t:Clean
-    C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe "$source\NancyBenchmark.csproj" /p:Configuration=Release /p:Platform="AnyCPU" /p:DeployOnBuild=true /p:PublishProfile=IIS
+    Exec { & $msbuild "$source\NancyBenchmark.csproj" /p:Configuration=Release /p:Platform="AnyCPU" /t:Clean }
+    Exec { & $msbuild "$source\NancyBenchmark.csproj" /p:Configuration=Release /p:Platform="AnyCPU" /p:DeployOnBuild=true /p:PublishProfile=IIS }
 }

nancy/setup_iis.py

@@ -9,7 +9,7 @@ def start(args):
   
   try:
     setup_util.replace_text("nancy/src/Web.config", "localhost", args.database_host)
-    subprocess.check_call("powershell -File setup_iis.ps1 start", cwd="nancy")
+    subprocess.check_call("powershell -Command .\\setup_iis.ps1 start", cwd="nancy")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,5 +18,5 @@ def stop():
   if os.name != 'nt':
     return 0
   
-  subprocess.Popen("powershell -File setup_iis.ps1 stop", cwd="nancy")
+  subprocess.check_call("powershell -Command .\\setup_iis.ps1 stop", cwd="nancy")
   return 0

servicestack/setup_iis.ps1

@@ -1,5 +1,23 @@
 param($action)
 
+$ErrorActionPreference = 'Stop'
+
+# From http://zduck.com/2012/powershell-batch-files-exit-codes/
+function Exec
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Position=0, Mandatory=1)]
+        [scriptblock]$Command,
+        [Parameter(Position=1, Mandatory=0)]
+        [string]$ErrorMessage = "Execution of command failed.`n$Command"
+    )
+    & $Command
+    if ($LastExitCode -ne 0) {
+        throw "Exec: $ErrorMessage"
+    }
+}
+
 $wwwroot = "C:\FrameworkBenchmarks\servicestack\www"
 $source = "C:\FrameworkBenchmarks\servicestack\src"
 $msbuild = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe"
@@ -16,6 +34,6 @@ if ($action -eq 'start') {
     New-WebSite -Name Benchmarks -Port 8080 -PhysicalPath $wwwroot
     
     # Build the project
-    &$msbuild "$source\ServiceStackBenchmark.csproj" /p:Configuration=Release /p:Platform="x64" /t:Clean
-    &$msbuild "$source\ServiceStackBenchmark.csproj" /p:Configuration=Release /p:Platform="x64" /p:DeployOnBuild=true /p:PublishProfile=IIS
+    Exec { & $msbuild "$source\ServiceStackBenchmark.csproj" /p:Configuration=Release /p:Platform="x64" /t:Clean }
+    Exec { & $msbuild "$source\ServiceStackBenchmark.csproj" /p:Configuration=Release /p:Platform="x64" /p:DeployOnBuild=true /p:PublishProfile=IIS }
 }

servicestack/setup_iis.py

@@ -9,7 +9,7 @@ def start(args):
   
   try:
     setup_util.replace_text("servicestack/src/Web.config", "localhost", args.database_host)
-    subprocess.check_call("powershell -File setup_iis.ps1 start", cwd="servicestack")
+    subprocess.check_call("powershell -Command .\\setup_iis.ps1 start", cwd="servicestack")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,5 +18,5 @@ def stop():
   if os.name != 'nt':
     return 0
   
-  subprocess.Popen("powershell -File setup_iis.ps1 stop", cwd="servicestack")
+  subprocess.check_call("powershell -Command .\\setup_iis.ps1 stop", cwd="servicestack")
   return 0

servicestack/setup_self.ps1

@@ -1,14 +1,32 @@
 param($action)
 
+$ErrorActionPreference = 'Stop'
+
+# From http://zduck.com/2012/powershell-batch-files-exit-codes/
+function Exec
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Position=0, Mandatory=1)]
+        [scriptblock]$Command,
+        [Parameter(Position=1, Mandatory=0)]
+        [string]$ErrorMessage = "Execution of command failed.`n$Command"
+    )
+    & $Command
+    if ($LastExitCode -ne 0) {
+        throw "Exec: $ErrorMessage"
+    }
+}
+
 $source = "C:\FrameworkBenchmarks\servicestack\src"
 $msbuild = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe"
 
 # Stop
-Stop-Process -Name servicestack -ErrorAction SilentlyContinue
+Get-Process | Where-Object { $_.Name -ieq "servicestack" } | Stop-Process
 
 if ($action -eq 'start') {
     # Build the project
-    &$msbuild "$source\ServiceStackBenchmark.sln" /p:DownloadNuGetExe=true /p:RequireRestoreConsent=false /p:Configuration=Release /p:Platform="x64" /t:Rebuild
+    Exec { & $msbuild "$source\ServiceStackBenchmark.sln" /p:DownloadNuGetExe=true /p:RequireRestoreConsent=false /p:Configuration=Release /p:Platform="x64" /t:Rebuild }
         
     Start-Process "$source\SelfHost\bin\Release\ServiceStackBenchmark.SelfHost.exe http://*:8080"
 }

servicestack/setup_self.py

@@ -9,7 +9,7 @@ def start(args):
   
   try:
     setup_util.replace_text("servicestack/svc/SelfHost/App.config", "localhost", args.database_host)
-    subprocess.check_call("powershell -File setup_self.ps1 start", cwd="servicestack")
+    subprocess.check_call("powershell -Command .\\setup_self.ps1 start", cwd="servicestack")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,5 +18,5 @@ def stop():
   if os.name != 'nt':
     return 0
   
-  subprocess.Popen("powershell -File setup_self.ps1 stop", cwd="servicestack")
+  subprocess.check_call("powershell -Command .\\setup_self.ps1 stop", cwd="servicestack")
   return 0

toolset/benchmark/benchmarker.py

@@ -10,6 +10,7 @@ import pprint
 import csv
 import sys
 import logging
+import socket
 from multiprocessing import Process
 from datetime import datetime
 
@@ -493,7 +494,16 @@ class Benchmarker:
 		      sudo /etc/init.d/postgresql restart
         """)
         time.sleep(10)
-        
+
+        if self.__is_port_bound(test.port):
+          self.__write_intermediate_results(test.name, "port " + str(test.port) + " is not available before start")
+          print textwrap.dedent("""
+            ---------------------------------------------------------
+              Error: Port {port} is not available before start {name}
+            ---------------------------------------------------------
+            """.format(name=test.name, port=str(test.port)))
+          return
+
         result = test.start()
         if result != 0: 
           test.stop()
@@ -535,6 +545,16 @@ class Benchmarker:
         ##########################
         test.stop()
         time.sleep(5)
+
+        if self.__is_port_bound(test.port):
+          self.__write_intermediate_results(test.name, "port " + str(test.port) + " was not released by stop")
+          print textwrap.dedent("""
+            -----------------------------------------------------
+              Error: Port {port} was not released by stop {name}
+            -----------------------------------------------------
+            """.format(name=test.name, port=str(test.port)))
+          return
+
         print textwrap.dedent("""
         -----------------------------------------------------
           Stopped {name}
@@ -582,6 +602,43 @@ class Benchmarker:
   # End __run_tests
   ############################################################
 
+  ############################################################
+  # __is_port_bound
+  # Check if the requested port is available. If it
+  # isn't available, then a previous test probably didn't
+  # shutdown properly.
+  ############################################################
+  def __is_port_bound(self, port):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+      # Try to bind to all IP addresses, this port
+      s.bind(("", port))
+      # If we get here, we were able to bind successfully,
+      # which means the port is free.
+    except:
+      # If we get an exception, it might be because the port is still bound
+      # which would be bad, or maybe it is a privileged port (<1024) and we
+      # are not running as root, or maybe the server is gone, but sockets are
+      # still in TIME_WAIT (SO_REUSEADDR). To determine which scenario, try to
+      # connect.
+      try:
+        s.connect(("127.0.0.1", port))
+        # If we get here, we were able to connect to something, which means
+        # that the port is still bound.
+        return True
+      except:
+        # An exception means that we couldn't connect, so a server probably
+        # isn't still running on the port.
+        pass
+    finally:
+      s.close()
+
+    return False
+
+  ############################################################
+  # End __is_port_bound
+  ############################################################
+
   ############################################################
   # __parse_results
   # Ensures that the system has all necessary software to run