Merge branch 'python-update' of https://github.com/methane/FrameworkBenchmarks into methane-python-update

config/benchmark_profile

@@ -11,6 +11,8 @@ export PLAY1_HOME=~/FrameworkBenchmarks/installs/play-1.2.5
 export MAVEN_HOME=~/FrameworkBenchmarks/installs/apache-maven-3.0.5
 export PERL_HOME=/opt/ActivePerl-5.16
 export DART_HOME=~/FrameworkBenchmarks/installs/dart-sdk
+export PYTHON_HOME=~/FrameworkBenchmarks/installs/python-2.7.5
+export PATH="$PYTHON_HOME/bin:$PATH"
 export PATH="$JAVA_HOME/bin:$GRAILS_HOME/bin:$PLAY_HOME:$PLAY1_HOME:$VERTX_HOME/bin:$GOROOT/bin:$NODE_HOME/bin:$HOME/FrameworkBenchmarks/installs/bin:$MAVEN_HOME/bin:$PERL_HOME/bin:$PERL_HOME/site/bin:$DART_HOME/bin:$PATH"
 
 export LD_LIBRARY_PATH='$LD_LIBRARY_PATH:/usr/local/apr/lib'

django-stripped/hello/hello/settings.py

@@ -15,8 +15,9 @@ DATABASES = {
         'NAME': 'hello_world',                      # Or path to database file if using sqlite3.
         'USER': 'benchmarkdbuser',                      # Not used with sqlite3.
         'PASSWORD': 'benchmarkdbpass',                  # Not used with sqlite3.
-        'HOST': 'localhost',                      # Set to empty string for localhost. Not used with sqlite3.
+        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
         'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+        'CONN_MAX_AGE': 30,
     }
 }
 
@@ -168,3 +169,5 @@ LOGGING = {
         },
     }
 }
+
+ALLOWED_HOSTS = ['*']

django-stripped/setup.py

@@ -1,13 +1,13 @@
-
 import subprocess
+import multiprocessing
 import sys
 import setup_util
 import os
 
 def start(args):
   setup_util.replace_text("django-stripped/hello/hello/settings.py", "HOST': '.*'", "HOST': '" + args.database_host + "'")
-
-  subprocess.Popen("gunicorn hello.wsgi:application --worker-class=\"egg:meinheld#gunicorn_worker\" -b 0.0.0.0:8080 -w " + str((args.max_threads * 2)) + " --log-level=critical", shell=True, cwd="django-stripped/hello")
+  subprocess.Popen("gunicorn hello.wsgi:application --worker-class=\"egg:meinheld#gunicorn_worker\" -b 0.0.0.0:8080 -w " +
+                   str((multiprocessing.cpu_count() * 3)) + " --log-level=critical", shell=True, cwd="django-stripped/hello")
   return 0
 def stop():
   p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)

django/hello/hello/settings.py

@@ -11,13 +11,13 @@ MANAGERS = ADMINS
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django_psycopg2_pool.gevent', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME': 'hello_world',                      # Or path to database file if using sqlite3.
         'USER': 'benchmarkdbuser',                      # Not used with sqlite3.
         'PASSWORD': 'benchmarkdbpass',                  # Not used with sqlite3.
         'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
         'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
-	'POOL_SIZE' : 32,
+        'CONN_MAX_AGE': 30,
     }
 }
 
@@ -167,3 +167,5 @@ LOGGING = {
         },
     }
 }
+
+ALLOWED_HOSTS = ['*']

django/hello/world/models.py

@@ -8,6 +8,6 @@ class World(models.Model):
     db_table = 'world'
 
 class Fortune(models.Model):
-  message = models.CharField()
+  message = models.CharField(max_length=65535)
   class Meta:
     db_table = 'fortune'

django/setup.py

@@ -1,4 +1,5 @@
 import subprocess
+import multiprocessing
 import sys
 import setup_util
 import os
@@ -9,11 +10,8 @@ home = expanduser("~")
 def start(args):
   setup_util.replace_text("django/hello/hello/settings.py", "HOST': '.*'", "HOST': '" + args.database_host + "'")
   setup_util.replace_text("django/hello/hello/settings.py", "\/home\/ubuntu",  home)
-  # because pooling doesn't work with meinheld, it's necessary to create a ton of gunicorn threads (think apache pre-fork)
-  # to allow the OS to switch processes when waiting for socket I/O.
-  #args.max_threads *= 8
-  # and go from there until the database server runs out of memory for new threads (connections)
-  subprocess.Popen("gunicorn hello.wsgi:application --worker-class=\"egg:meinheld#gunicorn_worker\"  -b 0.0.0.0:8080 -w " + str((args.max_threads * 8)) + " --log-level=critical", shell=True, cwd="django/hello")
+  subprocess.Popen("gunicorn hello.wsgi:application --worker-class=\"egg:meinheld#gunicorn_worker\"  -b 0.0.0.0:8080 -w " +
+                   str((multiprocessing.cpu_count() * 3)) + " --log-level=critical", shell=True, cwd="django/hello")
   return 0
 def stop():
   p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)

flask/setup_pypy.py

@@ -8,7 +8,7 @@ proc = None
 def start(args):
   global proc
   setup_util.replace_text("flask/app.py", "DBHOSTNAME", args.database_host)
-  proc = subprocess.Popen("~/FrameworkBenchmarks/installs/pypy-2.0/bin/pypy run_pypy.py --port=8080 --logging=error", shell=True, cwd="flask")
+  proc = subprocess.Popen("~/FrameworkBenchmarks/installs/pypy-2.0.2/bin/pypy run_pypy.py --port=8080 --logging=error", shell=True, cwd="flask")
   return 0
 
 def stop():

installer.py

@@ -26,7 +26,7 @@ class Installer:
     #######################################
     self.__run_command("sudo apt-get update", True)
     self.__run_command("sudo apt-get upgrade", True)
-    self.__run_command("sudo apt-get install build-essential libpcre3 libpcre3-dev libpcrecpp0 libssl-dev zlib1g-dev python-software-properties unzip git-core libcurl4-openssl-dev libbz2-dev libmysqlclient-dev mongodb-clients libreadline6-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt-dev libgdbm-dev ncurses-dev automake libffi-dev htop libtool bison libevent-dev libgstreamer-plugins-base0.10-0 libgstreamer0.10-0 liborc-0.4-0 libwxbase2.8-0 libwxgtk2.8-0 libgnutls-dev libjson0-dev libmcrypt-dev libicu-dev cmake gettext curl", True)
+    self.__run_command("sudo apt-get install build-essential libpcre3 libpcre3-dev libpcrecpp0 libssl-dev zlib1g-dev python-software-properties unzip git-core libcurl4-openssl-dev libbz2-dev libmysqlclient-dev mongodb-clients libreadline6-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt-dev libgdbm-dev ncurses-dev automake libffi-dev htop libtool bison libevent-dev libgstreamer-plugins-base0.10-0 libgstreamer0.10-0 liborc-0.4-0 libwxbase2.8-0 libwxgtk2.8-0 libgnutls-dev libjson0-dev libmcrypt-dev libicu-dev cmake gettext curl libpq-dev", True)
     self.__run_command("sudo add-apt-repository ppa:ubuntu-toolchain-r/test", True)
     self.__run_command("sudo apt-get update", True)
     self.__run_command("sudo apt-get install gcc-4.8 g++-4.8", True)
@@ -54,27 +54,32 @@ class Installer:
     #
     # Python
     #
+    
+    # .profile is not loaded yet. So we should use full path.
+    pypy_bin   = "~/FrameworkBenchmarks/installs/pypy-2.0.2/bin"
+    python_bin = "~/FrameworkBenchmarks/installs/python-2.7.5/bin"
+
+    self.__run_command("curl -L http://bitbucket.org/pypy/pypy/downloads/pypy-2.0.2-linux64.tar.bz2 | tar xj")
+    self.__run_command("curl http://www.python.org/ftp/python/2.7.5/Python-2.7.5.tgz | tar xz")
+    self.__run_command("./configure --prefix=$HOME/FrameworkBenchmarks/installs/python-2.7.5 --disable-shared", cwd="Python-2.7.5")
+    self.__run_command("make -j", cwd="Python-2.7.5")
+    self.__run_command("make install", cwd="Python-2.7.5")
+
+    self.__run_command("curl -L https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.7.1.tar.gz | tar xz")
+    self.__run_command(pypy_bin + "/pypy setup.py install", cwd="setuptools-0.7.1")
+    self.__run_command(python_bin + "/python setup.py install", cwd="setuptools-0.7.1")
+
+    self.__run_command("curl https://pypi.python.org/packages/source/p/pip/pip-1.3.1.tar.gz | tar xz")
+    self.__run_command(pypy_bin + "/pypy setup.py install", cwd="pip-1.3.1")
+    self.__run_command(python_bin + "/python setup.py install", cwd="pip-1.3.1")
 
-    self.__run_command("curl -L http://bitbucket.org/pypy/pypy/downloads/pypy-2.0-linux64.tar.bz2 | tar xvj")
-    self.__run_command("curl http://www.python.org/ftp/python/2.7.4/Python-2.7.4.tgz | tar xvz")
-    self.__run_command("./configure", cwd="Python-2.7.4")
-    self.__run_command("make -j", cwd="Python-2.7.4")
-    self.__run_command("sudo make install", cwd="Python-2.7.4")
-    self.__run_command("curl https://pypi.python.org/packages/source/d/distribute/distribute-0.6.38.tar.gz | tar xvz")
-    # run pypy before python. (`setup.py install` fails after `sudo setup.py install`)
-    self.__run_command("../pypy-2.0/bin/pypy setup.py install", cwd="distribute-0.6.38")
-    self.__run_command("sudo python setup.py install", cwd="distribute-0.6.38")
-    self.__run_command("curl https://pypi.python.org/packages/source/p/pip/pip-1.3.1.tar.gz | tar xvz")
-    self.__run_command("../pypy-2.0/bin/pypy setup.py install", cwd="pip-1.3.1")
-    self.__run_command("sudo python setup.py install", cwd="pip-1.3.1")
-    self.__run_command("sudo pip install MySQL-python==1.2.4")
-    self.__run_command("sudo pip install simplejson==3.0.7")
-    self.__run_command("curl http://initd.org/psycopg/tarballs/PSYCOPG-2-5/psycopg2-2.5.tar.gz | tar xvz")
-    self.__run_command("sudo python setup.py install", cwd="psycopg2-2.5")
-    self.__run_command("git clone https://github.com/iiilx/django-psycopg2-pool.git")
-    self.__run_command("sudo python setup.py install", cwd="django-psycopg2-pool")
-    self.__run_command("sudo pip install --upgrade numpy==1.7.1")
-    self.__run_command("pypy-2.0/bin/pip install PyMySQL==0.5")
+    self.__run_command(python_bin + "/pip install MySQL-python==1.2.4")
+    self.__run_command(python_bin + "/pip install simplejson==3.3.0")
+    self.__run_command("curl http://initd.org/psycopg/tarballs/PSYCOPG-2-5/psycopg2-2.5.tar.gz | tar xz")
+    self.__run_command(python_bin + "/python setup.py install", cwd="psycopg2-2.5")
+    self.__run_command(python_bin + "/pip install --upgrade numpy==1.7.1")
+    self.__run_command(pypy_bin + "/pip install PyMySQL==0.5")
+    self.__run_command(python_bin + "/easy_install -U 'ujson==1.30'")
 
     #
     # nodejs
@@ -221,17 +226,15 @@ class Installer:
     # Gunicorn
     #
 
-    self.__run_command("sudo easy_install -U 'gunicorn==0.17.4'")
-    self.__run_command("sudo pip install --upgrade meinheld")
-    self.__run_command("sudo easy_install -U 'eventlet==0.12.1'")
-    self.__run_command("sudo pip install --upgrade 'gevent==0.13.8'")
+    self.__run_command(python_bin + "/easy_install -U 'gunicorn==0.17.4'")
+    self.__run_command(python_bin + "/easy_install -U meinheld")
 
     #
     # Resin
     #
 
     self.__run_command("sudo cp -r /usr/lib/jvm/java-1.7.0-openjdk-amd64/include /usr/lib/jvm/java-1.7.0-openjdk-amd64/jre/bin/")
-    self.__run_command("curl http://www.caucho.com/download/resin-4.0.36.tar.gz | tar xvz")
+    self.__run_command("curl http://www.caucho.com/download/resin-4.0.36.tar.gz | tar xz")
     self.__run_command("./configure --prefix=`pwd`", cwd="resin-4.0.36")
     self.__run_command("make", cwd="resin-4.0.36")
     self.__run_command("make install", cwd="resin-4.0.36")
@@ -248,16 +251,13 @@ class Installer:
     # Tornado
     ##############################
     packages = "tornado==3.0.1 motor==0.1 pymongo==2.5"
-    self.__run_command("sudo pip install " + packages)
-    self.__run_command("pypy-2.0/bin/pip install " + packages)
+    self.__run_command(python_bin + "/pip install " + packages)
+    self.__run_command(pypy_bin   + "/pip install " + packages)
 
     ##############################
     # Django
     ##############################
-    self.__run_command("curl http://www.djangoproject.com/m/releases/1.4/Django-1.4.tar.gz | tar xvz")
-    self.__run_command("sudo rm -rf /usr/local/lib/python2.7/site-packages/django")
-    self.__run_command("sudo python setup.py install", cwd="Django-1.4")
-    self.__run_command("sudo easy_install -U 'ujson==1.30'")
+    self.__run_command(python_bin + "/pip install -U https://www.djangoproject.com/download/1.6a1/tarball/")
 
     ##############################
     # Grails
@@ -270,14 +270,14 @@ class Installer:
     ##############################
     # Flask
     ##############################
-    packages = "flask==0.9 flask-sqlalchemy==0.16 sqlalchemy==0.8.1 jinja2==2.6 werkzeug==0.8.3"
-    self.__run_command("sudo pip install " + packages)
-    self.__run_command("pypy-2.0/bin/pip install " + packages)
+    packages = "flask==0.9 flask-sqlalchemy==0.16 sqlalchemy==0.8.1 jinja2==2.7 werkzeug==0.8.3"
+    self.__run_command(python_bin + "/pip install " + packages)
+    self.__run_command(pypy_bin + "/pip install " + packages)
 
     ##############################
     # Bottle
     ##############################
-    self.__run_command("sudo pip install bottle bottle-sqlalchemy")
+    self.__run_command(python_bin + "/pip install bottle bottle-sqlalchemy")
 
     ##############################
     # Play 2

tornado/benchmark_config

@@ -13,6 +13,14 @@
       "query_url": "/db?queries=",
       "port": 8080,
       "sort": 53
+    },
+    "pypy-raw": {
+      "setup_file": "setup_pypy",
+      "json_url": "/json",
+      "db_url": "/db",
+      "query_url": "/db?queries=",
+      "port": 8080,
+      "sort": 134
     }
   }]
 }

tornado/server.py

@@ -13,11 +13,17 @@ tornado.options.define('port', default=8888, type=int, help=(
     "Server port"))
 
 class JsonSerializeTestHandler(tornado.web.RequestHandler):
+    def compute_etag(self):
+        return None
+
     def get(self):
         obj = dict(message="Hello, World!")
         self.write(obj)
 
 class QueryTestHandler(tornado.web.RequestHandler):
+    def compute_etag(self):
+        return None
+
     @tornado.web.asynchronous
     @gen.coroutine
     def get(self):

tornado/setup_pypy.py

@@ -0,0 +1,31 @@
+import subprocess
+import sys
+import setup_util
+import os
+from os.path import expanduser
+
+home = expanduser("~")
+cwd = "%s/FrameworkBenchmarks/tornado" % home
+pypy = "%s/FrameworkBenchmarks/installs/pypy-2.0.2/bin/pypy" % home
+
+
+def start(args):
+    setup_util.replace_text(
+        cwd + "/server.py", "localhost", args.database_host)
+
+    subprocess.Popen(pypy + " %s/FrameworkBenchmarks/tornado/server.py --port=8080 --logging=error" % home, shell=True, cwd=cwd)
+    return 0
+
+
+def stop():
+    p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
+    out, err = p.communicate()
+    for line in out.splitlines():
+        if 'server.py' in line:
+            #try:
+            pid = int(line.split(None, 2)[1])
+            os.kill(pid, 9)
+            #except OSError:
+            #    pass
+
+    return 0