make extension and more output

ULib/bash_profile.sh

@@ -1,26 +1,18 @@
-if [ -z "${FWROOT}" ]; then
-  export FWROOT=${HOME}/FrameworkBenchmarks
-fi
-#--------------------------------------
+#---------------------------------------------------------------------------------------------------------
+# bash_profile.sh - set the environment of our ULib installation
+#---------------------------------------------------------------------------------------------------------
 # TROOT - Path of this test's directory
-#--------------------------------------
-if [ -z "${TROOT}" ]; then
-  export TROOT=${FWROOT}/ULib
-fi
-#------------------------------------------------------------ --------------------------------------------
 # IROOT - Path of this test's install directory ($FWROOT/installs or $FWROOT/installs/pertest/<test-name>)
-#------------------------------------------------------------ --------------------------------------------
-if [ -z "${IROOT}" ]; then
-  export IROOT=${FWROOT}/installs
-fi
-
-# Set the root of our ULib installation
-export ULIB_ROOT=${IROOT}/ULib
-
-# Where to find the userver_tcp executable
-export PATH="${ULIB_ROOT}/bin:$PATH"
-
+#---------------------------------------------------------------------------------------------------------
+# INFO:root:Running installation for ULib
+# INSTALL: 
+#    export TROOT=$FWROOT/ULib && 
+#    export IROOT=$FWROOT/installs && 
+#    . $FWROOT/toolset/setup/linux/bash_functions.sh && 
+#    . $FWROOT/ULib/install.sh (cwd=$FWROOT//installs)
+#---------------------------------------------------------------------------------------------------------
 export ULIB_VERSION=1.4.1
+export ULIB_ROOT=${IROOT}/ULib
 export ULIB_DOCUMENT_ROOT=${ULIB_ROOT}/ULIB_DOCUMENT_ROOT
 export ULIB_BUILD_OUTPUT=${ULIB_ROOT}/ULIB_BUILD_OUTPUT.txt
 export ULIB_SERVER_OUTPUT=${ULIB_ROOT}/ULIB_SERVER_OUTPUT.txt

ULib/install.sh

@@ -1,27 +1,26 @@
 #!/bin/bash
 
-# ---------------------------------------------------------------------------------------------------------------------------------
-# toolset/run-tests.py --install server --install-strategy pertest --install-error-action continue --test ULib --type all --verbose
-# ---------------------------------------------------------------------------------------------------------------------------------
+# install.sh
+# --------------------------------------------------------------------------------------------------------
+# toolset/run-tests.py --install server --test ULib --type all --verbose
+# --------------------------------------------------------------------------------------------------------
+# TROOT - Path of this test's directory
+# IROOT - Path of this test's install directory ($FWROOT/installs or $FWROOT/installs/pertest/<test-name>)
+# --------------------------------------------------------------------------------------------------------
 # INFO:root:Running installation for ULib
 # INSTALL: 
 #    export TROOT=$FWROOT/ULib && 
 #    export IROOT=$FWROOT/installs && 
 #    . $FWROOT/toolset/setup/linux/bash_functions.sh && 
 #    . $FWROOT/ULib/install.sh (cwd=$FWROOT//installs)
-# ---------------------------------------------------------------------------------------------------------------------------------
-
-if [ -z "$TROOT" ]; then
-   return 1
-fi
-
+# --------------------------------------------------------------------------------------------------------
 . ${TROOT}/bash_profile.sh
 
 if [ ! -d "$ULIB_ROOT" ]; then
   mkdir -p $ULIB_ROOT
 fi
 
-cd ${ULIB_ROOT}
+cd $ULIB_ROOT
 
 if [ ! -f "benchmark.cfg" ]; then
   cat <<EOF >benchmark.cfg
@@ -30,14 +29,14 @@ userver {
  PREFORK_CHILD 8
  LISTEN_BACKLOG 8192
  MAX_KEEP_ALIVE 8192
- DOCUMENT_ROOT ${ULIB_DOCUMENT_ROOT}
+ DOCUMENT_ROOT $ULIB_DOCUMENT_ROOT
  PID_FILE ${ULIB_ROOT}/userver_tcp.pid
 }
 EOF
 fi
 
 if [ -x "bin/userver_tcp" ] && [ -x "${ULIB_DOCUMENT_ROOT}/db.so" ]; then
-  return 0
+  exit 0
 fi
 
 # 1. Download ULib
@@ -45,7 +44,7 @@ if [ ! -f "v${ULIB_VERSION}.tar.gz" ]; then
 	wget -nc --no-check-certificate --trust-server-names -O v${ULIB_VERSION}.tar.gz \
         https://github.com/stefanocasazza/ULib/archive/v${ULIB_VERSION}.tar.gz 2>/dev/null
 
-	tar xvf v${ULIB_VERSION}.tar.gz 2>/dev/null
+	tar xf v${ULIB_VERSION}.tar.gz 2>/dev/null
 fi
 
 cd ULib-$ULIB_VERSION
@@ -73,7 +72,7 @@ cd src/ulib/net/server/plugin/usp
 make -j1 db.la fortunes.la json.la plaintext.la queries.la updates.la >>$ULIB_BUILD_OUTPUT 2>&1
 
 if [ ! -e .libs/db.so ]; then
-   return 1
+   exit 1
 fi
 
 mkdir -p $ULIB_DOCUMENT_ROOT

ULib/setup.py

@@ -1,6 +1,56 @@
-# ---------------------------------------------------------------------------------------------------------------------------------
-# toolset/run-tests.py --install server --install-strategy pertest --install-error-action continue --test ULib --type all --verbose
-# ---------------------------------------------------------------------------------------------------------------------------------
+# setup.py
+
+# THIS CALLING IS WORKING
+# ----------------------------------------------------------------------
+# toolset/run-tests.py --install server --test ULib --type all --verbose
+# ----------------------------------------------------------------------
+# ....
+# ULib: setup.py running - FWROOT is /home/tfb/FrameworkBenchmarks
+# ....
+# INFO:root:Running installation for ULib
+# INSTALL: 
+#    export TROOT=$FWROOT/ULib && 
+#    export IROOT=$FWROOT/installs && 
+#    . $FWROOT/toolset/setup/linux/bash_functions.sh && 
+#    . $FWROOT/ULib/install.sh (cwd=$FWROOT//installs)
+# ....
+# -----------------------------------------------------
+#   Running Test: ULib ...
+# -----------------------------------------------------
+# ULib: TROOT is EMPTY
+# ULib: setup.py START - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
+# ULib: bash_profile.sh script START
+# ULib: trying to start server /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/bin/userver_tcp -c /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/benchmark.cfg
+# ULib: server STARTED
+# {'results': []}
+# ULib: TROOT is EMPTY
+# ULib: setup.py STOP - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
+# ULib: bash_profile.sh script START
+# ----------------------------------------------------------------------
+
+# NOW THIS CALLING IS WORKING TOO BUT IT IS CORRECT TO CALL IT BEFORE INSTALLING...?
+# ----------------------------------------------------------------------------------
+# toolset/run-tests.py --test ULib --type all --verbose
+# ----------------------------------------------------------------------------------
+# ....
+# ULib: setup.py running - FWROOT is /home/tfb/FrameworkBenchmarks
+# ....
+# -----------------------------------------------------
+#   Running Test: ULib ...
+# -----------------------------------------------------
+# ULib: TROOT is EMPTY
+# ULib: setup.py START - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
+# ULib: bash_profile.sh script START
+# ULib: NOT INSTALLED...
+# ULib: install.sh script START
+# ULib: trying to start server /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/bin/userver_tcp -c /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/benchmark.cfg
+# ULib: server STARTED
+# {'results': []}
+# ULib: TROOT is EMPTY
+# ULib: setup.py STOP - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
+# ULib: bash_profile.sh script START
+# -----------------------------------------------------
+
 import os
 import sys
 import time
@@ -11,90 +61,129 @@ import multiprocessing
 
 #log = logging.getLogger('framework_test')
 
-# --------------------------------------------------------------------------------------------------------
-# TROOT - Path of this test's directory
-# IROOT - Path of this test's install directory ($FWROOT/installs or $FWROOT/installs/pertest/<test-name>)
-# --------------------------------------------------------------------------------------------------------
-script = """
-if [ -n "${TROOT}" ] && [ -n "${IROOT}" ]; then
-  return 0
-fi
-return 1
+fwroot = setup_util.get_fwroot()
+
+print(    "ULib: setup.py running - FWROOT is %s" % fwroot)
+#log.info("ULib: setup.py running - FWROOT is %s" % fwroot)
+
+# Queries the shell for the value of IROOT
+def get_iroot():
+  try:
+    # Use printf to avoid getting a newline
+    # Redirect to avoid stderr printing
+    iroot = subprocess.check_output('printf \"$IROOT\" 2>/dev/null', shell=True, executable='/bin/bash')
+    if iroot != "":
+      return iroot
+    print(       'ULib: IROOT is EMPTY')
+#   log.critical('ULib: IROOT is EMPTY')
+    return setup_util.get_fwroot() + "/installs"
+  except subprocess.CalledProcessError:
+    return "";
+
+# Queries the shell for the value of TROOT
+def get_troot():
+  try:
+    # Use printf to avoid getting a newline
+    # Redirect to avoid stderr printing
+    troot = subprocess.check_output('printf \"$TROOT\" 2>/dev/null', shell=True, executable='/bin/bash')
+    if troot != "":
+      return troot
+    print(       'ULib: TROOT is EMPTY')
+#   log.critical('ULib: TROOT is EMPTY')
+    return setup_util.get_fwroot() + "/ULib"
+  except subprocess.CalledProcessError:
+    return "";
+
+def getEnvironmentVar(name, iroot, troot):
+  try:
+    script = """
+export IROOT=%s
+. %s/bash_profile.sh 2>>/tmp/ULib_setup.txt
+printf $%s           2>>/tmp/ULib_setup.txt
 """
+    value = subprocess.check_output(script % (iroot, troot, name), shell=True, executable='/bin/bash')
+    return value
+  except subprocess.CalledProcessError:
+    return "";
 
-p = subprocess.Popen(['sh'], stdin=subprocess.PIPE)
-p.communicate(script)
-if p.returncode != 0:
-  print('ULib: TROOT and IROOT are EMPTY...')
-# log.critical('ULib: TROOT and IROOT are EMPTY...')
-else:
-  script = """
-. ${TROOT}/bash_profile.sh
-if [ -x "${ULIB_ROOT}/bin/userver_tcp" ] && [ -e "${ULIB_DOCUMENT_ROOT}/db.so" ] && [ -f "${ULIB_ROOT}/benchmark.cfg" ]; then
-  return 0
+def callScriptAndCheckForInstallation(name, iroot, troot):
+  try:
+    script = """
+export IROOT=%s
+export TROOT=%s
+. %s/%s.sh >>/tmp/ULib_setup.txt 2>&1
+if [ ! -x "${ULIB_ROOT}/bin/userver_tcp" ] || [ ! -e "${ULIB_DOCUMENT_ROOT}/db.so" ] || [ ! -f "${ULIB_ROOT}/benchmark.cfg" ]; then
+  exit 1
 fi
-return 1
 """
+    print(   "ULib: %s.sh script START" % name)
+#   log.info("ULib: %s.sh script START" % name)
+    return subprocess.call(script % (iroot, troot, troot, name), shell=True, executable='/bin/bash')
+  except subprocess.CalledProcessError:
+    return 1
 
-  p1 = subprocess.Popen(['sh'], stdin=subprocess.PIPE)
-  p1.communicate(script)
-  if p1.returncode != 0:
-    print('ULib install script is NOT BEEN CALLED, I call it now...')
-#   log.critical('ULib install script is NOT BEEN CALLED, I call it now...')
-    subprocess.call("${TROOT}/install.sh", shell=True, executable='/bin/bash')
-    p2 = subprocess.Popen(['sh'], stdin=subprocess.PIPE)
-    p2.communicate(script)
-    if p2.returncode != 0:
-      print('ULib install script FAILED')
-#     log.critical('ULib install script FAILED')
-      ulib_root = subprocess.check_output('. ${TROOT}/bash_profile.sh; printf "$ULIB_ROOT" 2>/dev/null', shell=True, executable='/bin/bash')
-      fprg = ulib_root + "/bin/userver_tcp"
-      if not os.path.exists(fprg):
-        print("ULib server program " + fprg + " NOT EXIST")
-#       log.critical("ULib server program " + fprg + " NOT EXIST")
-      fcfg = ulib_root + "/benchmark.cfg"
-      if not os.path.exists(fcfg):
-        print("ULib configuration file " + fcfg + " NOT EXIST")
-#       log.critical("ULib configuration file " + fcfg + " NOT EXIST")
-      fusp = subprocess.check_output('. ${TROOT}/bash_profile.sh; printf "$ULIB_DOCUMENT_ROOT" 2>/dev/null', shell=True, executable='/bin/bash') + "/db.so"
-      if not os.path.exists(fusp):
-        print("ULib usp page " + fusp + " NOT EXIST")
-#       log.critical("ULib usp page " + fusp + " NOT EXIST")
-      print('Aborting')
-#     log.critical('Aborting')
-      exit(1)
+# --------------------------------------------------------------------------------------------------------
+# TROOT - Path of this test's directory
+# IROOT - Path of this test's install directory ($FWROOT/installs or $FWROOT/installs/pertest/<test-name>)
+# --------------------------------------------------------------------------------------------------------
+def checkEnvironment(iroot, troot):
+  try:
+    if callScriptAndCheckForInstallation('bash_profile', iroot, troot) != 0:
+      print(       'ULib: NOT INSTALLED...')
+#     log.critical('ULib: NOT INSTALLED...')
+      if callScriptAndCheckForInstallation('install', iroot, troot) != 0:
+        print(   "ULib: install.sh script FAILED")
+#       log.info("ULib: install.sh script FAILED")
+        return False
+    return True
+  except:
+    pass
+  return False
 
 ##############
 # start(args)
 ##############
 def start(args, logfile, errfile):
   try:
-    ulib_root = subprocess.check_output('. ${TROOT}/bash_profile.sh; printf "$ULIB_ROOT" 2>/dev/null', shell=True, executable='/bin/bash')
-    fprg = ulib_root + "/bin/userver_tcp"
+    iroot = get_iroot()
+    troot = get_troot()
+
+    print(   "ULib: setup.py START - IROOT is %s - TROOT is %s" % (iroot, troot))
+#   log.info("ULib: setup.py START - IROOT is %s - TROOT is %s" % (iroot, troot))
+
+    if not checkEnvironment(iroot, troot):
+      return 1
+
+    ulib_root = getEnvironmentVar('ULIB_ROOT', iroot, troot)
+
     fcfg = ulib_root + "/benchmark.cfg"
+
     # 1. Change ULib Server configuration
+
     # I don't understand if the two approach are different...
     #threads = str(args.max_threads)
     PROCS = str(multiprocessing.cpu_count())
     setup_util.replace_text(fcfg, "PREFORK_CHILD .*", "PREFORK_CHILD " + PROCS)
 
-    ulib_server_output = subprocess.check_output('. ${TROOT}/bash_profile.sh; printf "$ULIB_SERVER_OUTPUT" 2>/dev/null', shell=True, executable='/bin/bash')
+    fprg = ulib_root + "/bin/userver_tcp"
 
     # 2. Start ULib Server (userver_tcp)
-    print("trying to start ULib server " + fprg + " -c " + fcfg)
-#   log.info("trying to start ULib server " + fprg + " -c " + fcfg)
+    print(   "ULib: trying to start server " + fprg + " -c " + fcfg)
+#   log.info("ULib: trying to start server " + fprg + " -c " + fcfg)
 
     # sudo mysqlcheck -v -r -A -u benchmarkdbuser -p
     os.putenv("ORM_DRIVER","mysql")
     os.putenv("ORM_OPTION","host=" + args.database_host + " user=benchmarkdbuser password=benchmarkdbpass dbname=hello_world")
     os.putenv("UMEMPOOL", "1583,1507,-19,45,16458,523,-27,-14,27")
 
+    ulib_server_output = getEnvironmentVar('ULIB_SERVER_OUTPUT', iroot, troot)
+
     # Run in the background, but keep stdout/stderr for easy debugging
     subprocess.Popen(                      fprg + " -c " + fcfg + " >" + ulib_server_output + " 2>&1", shell=True, stdout=logfile, stderr=errfile)
   # subprocess.Popen("UTRACE=\"0 50M\" " + fprg + " -c " + fcfg + " >" + ulib_server_output + " 2>&1", shell=True, stdout=logfile, stderr=errfile)
 
-    print("ULib server STARTED")
-#   log.info("ULib server STARTED")
+    print(   "ULib: server STARTED")
+#   log.info("ULib: server STARTED")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -104,7 +193,17 @@ def start(args, logfile, errfile):
 ##############
 def stop(logfile, errfile):
   try:
-    ulib_root = subprocess.check_output('. ${TROOT}/bash_profile.sh; printf "$ULIB_ROOT" 2>/dev/null', shell=True, executable='/bin/bash')
+    iroot = get_iroot()
+    troot = get_troot()
+
+    print(   "ULib: setup.py STOP - IROOT is %s - TROOT is %s" % (iroot, troot))
+#   log.info("ULib: setup.py STOP - IROOT is %s - TROOT is %s" % (iroot, troot))
+
+    if not checkEnvironment(iroot, troot):
+        return 1
+
+    ulib_root = getEnvironmentVar('ULIB_ROOT', iroot, troot)
+
     # Stop ULib Server (userver_tcp)
     subprocess.check_call("kill -TERM $( cat " + ulib_root + "/userver_tcp.pid )", shell=True, stderr=errfile, stdout=logfile)
     time.sleep(2);
@@ -114,7 +213,7 @@ def stop(logfile, errfile):
        if 'userver_tcp' in line:
          pid = int(line.split(None, 2)[1])
          os.kill(pid, 9)
-    subprocess.call("rm -f " + ulib_root + "/userver_tcp.pid", shell=True, stderr=errfile, stdout=logfile)
+    subprocess.call("rm -f " + ulib_root + "/userver_tcp.pid /tmp/ULib_setup.txt", shell=True, stderr=errfile, stdout=logfile)
     return 0
   except subprocess.CalledProcessError:
     return 1