Refactoring bottle and nginx_uwsgi

bottle/app.py

@@ -1,26 +1,36 @@
+from functools import partial
+from operator import attrgetter, itemgetter
+from random import randint
+import os
+import sys
+
 from bottle import Bottle, route, request, run, template, response
 from bottle.ext import sqlalchemy 
 from sqlalchemy import create_engine, Column, Integer, Unicode
 from sqlalchemy.ext.declarative import declarative_base
-from random import randint
-import sys
-from operator import attrgetter, itemgetter
-from functools import partial
 
 try:
     import ujson as json
 except ImportError:
     import json
 
+if sys.version_info[0] == 3:
+    xrange = range
+
+
+DBHOSTNAME = os.environ.get('DBHOSTNAME', 'localhost')
+DATABASE_URI = 'mysql://benchmarkdbuser:benchmarkdbpass@%s:3306/hello_world?charset=utf8' % DBHOSTNAME
+
 app = Bottle()
-app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://benchmarkdbuser:benchmarkdbpass@DBHOSTNAME:3306/hello_world?charset=utf8'
 Base = declarative_base()
-db_engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
-plugin = sqlalchemy.Plugin(db_engine, keyword='db', )
+db_engine = create_engine(DATABASE_URI)
+plugin = sqlalchemy.Plugin(db_engine, keyword='db')
 app.install(plugin)
 
-if sys.version_info[0] == 3:
-    xrange = range
+# Engine for raw operation. Use autocommit.
+raw_engine = create_engine(DATABASE_URI,
+                           connect_args={'autocommit': True},
+                           pool_reset_on_return=None)
 
 
 class World(Base):
@@ -28,15 +38,14 @@ class World(Base):
   id = Column(Integer, primary_key=True)
   randomNumber = Column(Integer)
 
-  # http://stackoverflow.com/questions/7102754/jsonify-a-sqlalchemy-result-set-in-flask
-  @property
   def serialize(self):
      """Return object data in easily serializeable format"""
      return {
-         'id'         : self.id,
-         'randomNumber': self.randomNumber
+         'id': self.id,
+         'randomNumber': self.randomNumber,
      }
 
+
 class Fortune(Base):
   __tablename__ = "Fortune"
   id = Column(Integer, primary_key=True)
@@ -49,60 +58,77 @@ def hello():
     resp = {"message": "Hello, World!"}
     return json.dumps(resp)
 
+
 @app.route("/db")
+def get_random_world_single(db):
+    """Test Type 2: Single Database Query"""
+    wid = randint(1, 10000)
+    world = db.query(World).get(wid).serialize()
+    response.content_type = 'application/json'
+    return json.dumps(world)
+
+
[email protected]("/raw-db")
+def get_random_world_single_raw():
+    connection = raw_engine.connect()
+    wid = randint(1, 10000)
+    try:
+        result = connection.execute("SELECT id, randomNumber FROM world WHERE id = " + str(wid)).fetchone()
+        world = {'id': result[0], 'randomNumber': result[1]}
+        response.content_type = 'application/json'
+        return json.dumps(world)
+    finally:
+        connection.close()
+
+
[email protected]("/queries")
 def get_random_world(db):
+    """Test Type 3: Multiple database queries"""
     num_queries = request.query.get('queries', 1, type=int)
-    worlds = []
+    if num_queries > 500:
+        num_queries = 500
     rp = partial(randint, 1, 10000)
-    for i in xrange(num_queries):
-        worlds.append(db.query(World).get(rp()).serialize)
+    get = db.query(World).get
+    worlds = [get(rp()).serialize() for _ in xrange(num_queries)]
     response.content_type = 'application/json'
     return json.dumps(worlds)
 
[email protected]("/dbs")
-def get_random_world_single(db):
-    wid = randint(1, 10000)
-    world = db.query(World).get(wid).serialize
-    response.content_type = 'application/json'
-    return json.dumps(world)
-  
[email protected]("/dbraw")
+
[email protected]("/raw-queries")
 def get_random_world_raw():
-    connection = db_engine.connect()
     num_queries = request.query.get('queries', 1, type=int)
+    if num_queries > 500:
+        num_queries = 500
     worlds = []
     rp = partial(randint, 1, 10000)
-    for i in xrange(int(num_queries)):
-        result = connection.execute("SELECT * FROM world WHERE id = " + str(rp())).fetchone()
-        worlds.append({'id': result[0], 'randomNumber': result[1]})
-    connection.close()
+    connection = raw_engine.connect()
+    try:
+        for i in xrange(num_queries):
+            result = connection.execute("SELECT id, randomNumber FROM world WHERE id = " + str(rp())).fetchone()
+            worlds.append({'id': result[0], 'randomNumber': result[1]})
+    finally:
+        connection.close()
     response.content_type = 'application/json'
     return json.dumps(worlds)
 
[email protected]("/dbsraw")
-def get_random_world_single_raw():
-    connection = db_engine.connect()
-    wid = randint(1, 10000)
-    result = connection.execute("SELECT * FROM world WHERE id = " + str(wid)).fetchone()
-    worlds = {'id': result[0], 'randomNumber': result[1]}
-    connection.close()
-    response.content_type = 'application/json'
-    return json.dumps(worlds)
 
 @app.route("/fortune")
 def fortune_orm(db):
   fortunes=db.query(Fortune).all()
   fortunes.append(Fortune(id=0, message="Additional fortune added at request time."))
-  fortunes=sorted(fortunes, key=attrgetter('message'))
+  fortunes.sort(key=attrgetter('message'))
   return template('fortune-obj', fortunes=fortunes)
 
[email protected]("/fortuneraw")
+
[email protected]("/raw-fortune")
 def fortune_raw():
-    connection = db_engine.connect()
-    fortunes=[(f.id, f.message) for f in connection.execute("SELECT * FROM Fortune")]
-    fortunes.append((0, u'Additional fortune added at request time.'))
-    fortunes=sorted(fortunes, key=itemgetter(1))
-    connection.close()
+    connection = raw_engine.connect()
+    try:
+        fortunes=[(f.id, f.message) for f in connection.execute("SELECT * FROM Fortune")]
+        fortunes.append((0, u'Additional fortune added at request time.'))
+        fortunes=sorted(fortunes, key=itemgetter(1))
+    finally:
+        connection.close()
     return template('fortune', fortunes=fortunes)
 
 
@@ -120,7 +146,7 @@ def updates(db):
     for id in ids:
         world = db.query(World).get(id)
         world.randomNumber = rp()
-        worlds.append(world.serialize)
+        worlds.append(world.serialize())
 
     response.content_type = 'application/json'
     return json.dumps(worlds)
@@ -133,7 +159,7 @@ def raw_updates():
     if num_queries > 500:
         num_queries = 500
 
-    conn = db_engine.connect()
+    conn = raw_engine.connect()
 
     worlds = []
     rp = partial(randint, 1, 10000)

bottle/benchmark_config

@@ -4,8 +4,8 @@
     "default": {
       "setup_file": "setup",
       "json_url": "/json",
-      "db_url": "/dbs",
-      "query_url": "/db?queries=",
+      "db_url": "/db",
+      "query_url": "/queries?queries=",
       "fortune_url": "/fortune",
       "update_url": "/updates?queries=",
       "plaintext_url": "/plaintext",
@@ -16,19 +16,19 @@
       "framework": "bottle",
       "language": "Python",
       "orm": "Full",
-      "platform": "Gunicorn/Meinheld",
+      "platform": "Meinheld",
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "bottle",
-      "notes": "CPython 2.7",
+      "notes": "CPython 2",
       "versus": "wsgi"
     },
     "py3": {
       "setup_file": "setup_py3",
       "json_url": "/json",
-      "db_url": "/dbs",
-      "query_url": "/db?queries=",
+      "db_url": "/db",
+      "query_url": "/queries?queries=",
       "fortune_url": "/fortune",
       "update_url": "/updates?queries=",
       "plaintext_url": "/plaintext",
@@ -39,19 +39,19 @@
       "framework": "bottle",
       "language": "Python",
       "orm": "Full",
-      "platform": "Gunicorn/Meinheld",
+      "platform": "Meinheld",
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "bottle-py3",
-      "notes": "CPython 3.4",
+      "notes": "CPython 3",
       "versus": "wsgi"
     },
     "pypy": {
       "setup_file": "setup_pypy",
       "json_url": "/json",
-      "db_url": "/dbs",
-      "query_url": "/db?queries=",
+      "db_url": "/db",
+      "query_url": "/queries?queries=",
       "fortune_url": "/fortune",
       "update_url": "/updates?queries=",
       "plaintext_url": "/plaintext",
@@ -62,19 +62,19 @@
       "framework": "bottle",
       "language": "Python",
       "orm": "Full",
-      "platform": "Gunicorn/Tornado",
+      "platform": "Tornado",
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "bottle-pypy",
-      "notes": "PyPy 2.2",
+      "notes": "PyPy",
       "versus": "wsgi"
     },
     "mysql-raw": {
       "setup_file": "setup",
-      "db_url": "/dbsraw",
-      "query_url": "/dbraw?queries=",
-      "fortune_url": "/fortuneraw",
+      "db_url": "/raw-db",
+      "query_url": "/raw-queries?queries=",
+      "fortune_url": "/raw-fortune",
       "update_url": "/raw-updates?queries=",
       "port": 8080,
       "approach": "Realistic",
@@ -83,19 +83,19 @@
       "framework": "bottle",
       "language": "Python",
       "orm": "Raw",
-      "platform": "Gunicorn/Meinheld",
+      "platform": "Meinheld",
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "bottle-raw",
-      "notes": "CPython 2.7",
+      "notes": "CPython 2",
       "versus": "wsgi"
     },
     "nginx-uwsgi": {
       "setup_file": "setup_nginxuwsgi",
       "json_url": "/json",
-      "db_url": "/dbs",
-      "query_url": "/db?queries=",
+      "db_url": "/db",
+      "query_url": "/queries?queries=",
       "fortune_url": "/fortune",
       "update_url": "/updates?queries=",
       "plaintext_url": "/plaintext",
@@ -111,7 +111,7 @@
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "bottle-uWSGI",
-      "notes": "CPython 2.7",
+      "notes": "CPython 2",
       "versus": "wsgi"
     }
   }]

bottle/gunicorn_conf.py

@@ -1,11 +1,21 @@
 import multiprocessing
+import sys
+
+_is_pypy = hasattr(sys, 'pypy_version_info')
+
 
 workers = multiprocessing.cpu_count() * 3
 bind = "0.0.0.0:8080"
-worker_class = "meinheld.gmeinheld.MeinheldWorker"
 keepalive = 120
 
-def post_fork(server, worker):
-    # Disalbe access log
-    import meinheld.server
-    meinheld.server.set_access_logger(None)
+
+if _is_pypy:
+    worker_class = "tornado"
+else:
+    worker_class = "meinheld.gmeinheld.MeinheldWorker"
+
+    def post_fork(server, worker):
+        # Disalbe access log
+        import meinheld.server
+        meinheld.server.set_access_logger(None)
+

bottle/setup.py

@@ -1,20 +1,22 @@
 import subprocess
-import setup_util
 import os
 
-bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
+BIN = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
 
 proc = None
 
 
 def start(args, logfile, errfile):
     global proc
-    setup_util.replace_text("bottle/app.py", "DBHOSTNAME", args.database_host)
     proc = subprocess.Popen(
-        [bin_dir + "/gunicorn", "-c", "gunicorn_conf.py", "app:app"],
+        [BIN + "/gunicorn",
+         "-c", "gunicorn_conf.py",
+         "-e", "DBHOSTNAME=" + args.database_host,
+         "app:app"],
         cwd="bottle", stderr=errfile, stdout=logfile)
     return 0
 
+
 def stop(logfile, errfile):
     global proc
     if proc is None:

bottle/setup_nginxuwsgi.py

@@ -1,27 +1,36 @@
 import subprocess
 import multiprocessing
 import os
-import setup_util
+
 
 bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
 config_dir = os.path.expanduser('~/FrameworkBenchmarks/config')
 NCPU = multiprocessing.cpu_count()
+NGINX_COMMAND = '/usr/local/nginx/sbin/nginx -c ' + config_dir + '/nginx_uwsgi.conf'
+
 
 def start(args, logfile, errfile):
-    setup_util.replace_text("bottle/app.py", "DBHOSTNAME", args.database_host)
     try:
-        subprocess.check_call('sudo /usr/local/nginx/sbin/nginx -c ' +
-            config_dir + '/nginx_uwsgi.conf', shell=True, stderr=errfile, stdout=logfile)
+        subprocess.call(
+            NGINX_COMMAND,
+            shell=True, stdout=logfile, stderr=errfile)
+
         # Run in the background, but keep stdout/stderr for easy debugging
-        subprocess.Popen(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi.ini' +
-            ' --processes ' + str(NCPU * 3) +
-            ' --wsgi app:app',
+        subprocess.Popen(
+            "{0}/uwsgi --ini {1}/uwsgi.ini --processes {2} --env DBHOSTNAME={3} --wsgi app:app".format(
+                bin_dir, config_dir, NCPU*3, args.database_host),
             shell=True, cwd='bottle', stderr=errfile, stdout=logfile)
+
         return 0
     except subprocess.CalledProcessError:
         return 1
 
+
 def stop(logfile, errfile):
-    subprocess.call('sudo /usr/local/nginx/sbin/nginx -s stop', shell=True, stderr=errfile, stdout=logfile)
-    subprocess.call(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi_stop.ini', shell=True, stderr=errfile, stdout=logfile)
+    subprocess.call(
+        NGINX_COMMAND + ' -s stop',
+        shell=True, stdout=logfile, stderr=errfile)
+
+    subprocess.call(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi_stop.ini',
+                    shell=True, stderr=errfile, stdout=logfile)
     return 0

bottle/setup_py3.py

@@ -1,20 +1,22 @@
 import subprocess
-import setup_util
 import os
 
-bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py3/bin')
+BIN = os.path.expanduser('~/FrameworkBenchmarks/installs/py3/bin')
 
 proc = None
 
 
 def start(args, logfile, errfile):
     global proc
-    setup_util.replace_text("bottle/app.py", "DBHOSTNAME", args.database_host)
     proc = subprocess.Popen(
-        [bin_dir + "/gunicorn", "-c", "gunicorn_conf.py", "app:app"],
+        [BIN + "/gunicorn",
+         "-c", "gunicorn_conf.py",
+         "-e", "DBHOSTNAME=" + args.database_host,
+         "app:app"],
         cwd="bottle", stderr=errfile, stdout=logfile)
     return 0
 
+
 def stop(logfile, errfile):
     global proc
     if proc is None:

bottle/setup_pypy.py

@@ -1,27 +1,22 @@
 import subprocess
-import setup_util
-import multiprocessing
 import os
 
-bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/pypy/bin')
-NCPU = multiprocessing.cpu_count()
+BIN = os.path.expanduser('~/FrameworkBenchmarks/installs/pypy/bin')
 
 proc = None
 
 
 def start(args, logfile, errfile):
     global proc
-    setup_util.replace_text("bottle/app.py", "DBHOSTNAME", args.database_host)
-    proc = subprocess.Popen([
-        bin_dir + "/gunicorn",
-        "app:app",
-        "-k", "tornado",
-        "-b", "0.0.0.0:8080",
-        '-w', str(NCPU*3),
-        "--log-level=critical"],
+    proc = subprocess.Popen(
+        [BIN + "/gunicorn",
+         "-c", "gunicorn_conf.py",
+         "-e", "DBHOSTNAME=" + args.database_host,
+         "app:app"],
         cwd="bottle", stderr=errfile, stdout=logfile)
     return 0
 
+
 def stop(logfile, errfile):
     global proc
     if proc is None:
@@ -30,4 +25,3 @@ def stop(logfile, errfile):
     proc.wait()
     proc = None
     return 0
-

config/nginx_uwsgi.conf

@@ -3,11 +3,13 @@
 # One worker process per core
 worker_processes auto;
 error_log stderr error;
+pid /tmp/nginx.pid;
 
 events {
     # This needed to be increased because the nginx error log said so.
     # http://nginx.org/en/docs/ngx_core_module.html#worker_connections
     worker_connections  65535;
+    multi_accept on;
 }
 
 http {

flask/setup_nginxuwsgi.py

@@ -6,12 +6,16 @@ import setup_util
 bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
 config_dir = os.path.expanduser('~/FrameworkBenchmarks/config')
 NCPU = multiprocessing.cpu_count()
+NGINX_COMMAND = '/usr/local/nginx/sbin/nginx -c ' + config_dir + '/nginx_uwsgi.conf'
+
 
 def start(args, logfile, errfile):
     setup_util.replace_text("flask/app.py", "DBHOSTNAME", args.database_host)
     try:
-        subprocess.check_call('sudo /usr/local/nginx/sbin/nginx -c ' +
-            config_dir + '/nginx_uwsgi.conf', shell=True, stderr=errfile, stdout=logfile)
+        subprocess.call(
+            NGINX_COMMAND,
+            shell=True, stdout=logfile, stderr=errfile)
+
         # Run in the background, but keep stdout/stderr for easy debugging
         subprocess.Popen(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi.ini' +
             ' --processes ' + str(NCPU * 3) +
@@ -22,14 +26,11 @@ def start(args, logfile, errfile):
         return 1
 
 def stop(logfile, errfile):
-    subprocess.call('sudo /usr/local/nginx/sbin/nginx -s stop', shell=True, stderr=errfile, stdout=logfile)
+    subprocess.call(
+        NGINX_COMMAND + ' -s stop',
+        shell=True, stdout=logfile, stderr=errfile)
     subprocess.call(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi_stop.ini', shell=True, stderr=errfile, stdout=logfile)
 
-    p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
-    out, err = p.communicate()
-    for line in out.splitlines():
-      if 'FrameworkBenchmarks/installs/py2/bin/' in line:
-        pid = int(line.split(None,2)[1])
-        os.kill(pid, 15)
-
+    os.system('killall nginx')
+    os.system('killall uwsgi')
     return 0

uwsgi/setup_nginx.py

@@ -5,11 +5,15 @@ import os
 bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
 config_dir = os.path.expanduser('~/FrameworkBenchmarks/config')
 NCPU = multiprocessing.cpu_count()
+NGINX_COMMAND = '/usr/local/nginx/sbin/nginx -c ' + config_dir + '/nginx_uwsgi.conf'
+
 
 def start(args, logfile, errfile):
     try:
-        subprocess.check_call('sudo /usr/local/nginx/sbin/nginx -c ' +
-            config_dir + '/nginx_uwsgi.conf', shell=True)
+        subprocess.check_call(
+            NGINX_COMMAND,
+            shell=True, stdout=logfile, stderr=errfile)
+
         # Run in the background, but keep stdout/stderr for easy debugging.
         # Note that this uses --gevent 1000 just like setup.py.
         subprocess.Popen(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi.ini' +
@@ -22,6 +26,12 @@ def start(args, logfile, errfile):
         return 1
 
 def stop(logfile, errfile):
-    subprocess.call('sudo /usr/local/nginx/sbin/nginx -s stop', shell=True, stdout=logfile, stderr=errfile)
+    subprocess.check_call(
+        NGINX_COMMAND + ' -s stop',
+        shell=True, stdout=logfile, stderr=errfile)
+
     subprocess.call(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi_stop.ini', shell=True, stdout=logfile, stderr=errfile)
+
+    os.system('killall nginx')
+    os.system('killall uwsgi')
     return 0

wsgi/setup_nginxuwsgi.py

@@ -5,22 +5,30 @@ import os
 bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
 config_dir = os.path.expanduser('~/FrameworkBenchmarks/config')
 NCPU = multiprocessing.cpu_count()
+NGINX_COMMAND = '/usr/local/nginx/sbin/nginx -c ' + config_dir + '/nginx_uwsgi.conf'
 
 def start(args, logfile, errfile):
     try:
-        subprocess.check_call('sudo /usr/local/nginx/sbin/nginx -c ' +
-            config_dir + '/nginx_uwsgi.conf', shell=True)
+        subprocess.check_call(
+            NGINX_COMMAND,
+            shell=True, stdout=logfile, stderr=errfile)
+
         # Run in the background, but keep stdout/stderr for easy debugging
         subprocess.Popen(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi.ini' +
             ' --processes ' + str(NCPU) +
             ' --wsgi hello:app',
-            shell=True, cwd='wsgi',
-            stdout=logfile, stderr=errfile)
+            shell=True, cwd='wsgi', stdout=logfile, stderr=errfile)
         return 0
     except subprocess.CalledProcessError:
         return 1
 
 def stop(logfile, errfile):
-    subprocess.call('sudo /usr/local/nginx/sbin/nginx -s stop', shell=True, stdout=logfile, stderr=errfile)
+    subprocess.check_call(
+        NGINX_COMMAND + ' -s stop',
+        shell=True, stdout=logfile, stderr=errfile)
+
     subprocess.call(bin_dir + '/uwsgi --ini ' + config_dir + '/uwsgi_stop.ini', shell=True, stdout=logfile, stderr=errfile)
+
+    os.system('killall nginx')
+    os.system('killall uwsgi')
     return 0