Merge pull request #1422 from Eyepea/API-Hour_fine_tuning

API-Hour fine tuning

frameworks/Python/.gitignore

@@ -2,5 +2,5 @@ uwsgi_params
 *.pid
 
 # ignore virtual environments
-pyvenv/
-venv/
+pyvenv*/
+venv*/

frameworks/Python/API-Hour/benchmark_config

@@ -1,5 +1,5 @@
 {
-  "framework": "aiohttp.web",
+  "framework": "AsyncIO",
   "tests": [{
     "default": {
       "setup_file": "setup",
@@ -17,11 +17,65 @@
       "language": "Python",
       "orm": "Raw",
       "platform": "API-Hour",
-      "webserver": "None",
+      "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "API-Hour",
-      "notes": "Python 3.4 + AsyncIO"
+      "display_name": "API-Hour+aiohttp.web+pgsql",
+      "notes": "Python 3 + API-Hour + AsyncIO + aiohttp.web + PostgreSQL"
+    },
+    "redis": {
+      "setup_file": "setup",
+      "db_url": "/db_redis",
+      "query_url": "/queries_redis?queries=",
+      "fortune_url": "/fortunes_redis",
+      "update_url": "/updates_redis?queries=",
+      "port": 8008,
+      "approach": "Realistic",
+      "classification": "Micro",
+      "database": "Redis",
+      "framework": "aiohttp.web",
+      "language": "Python",
+      "orm": "Raw",
+      "platform": "API-Hour",
+      "webserver": "Gunicorn",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "API-Hour+aiohttp.web+redis",
+      "notes": "Python 3 + API-Hour + AsyncIO + aiohttp.web + Redis"
+    },
+    "json": {
+      "setup_file": "setup",
+      "json_url": "/json",
+      "port": 8009,
+      "approach": "Stripped",
+      "classification": "Platform",
+      "database": "None",
+      "framework": "AsyncIO",
+      "language": "Python",
+      "orm": "Raw",
+      "platform": "API-Hour",
+      "webserver": "Gunicorn",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "API-Hour+AsyncIO",
+      "notes": "Python 3 + API-Hour + AsyncIO"
+    },
+    "plaintext": {
+      "setup_file": "setup",
+      "plaintext_url": "/plaintext",
+      "port": 8011,
+      "approach": "Stripped",
+      "classification": "Platform",
+      "database": "None",
+      "framework": "AsyncIO",
+      "language": "Python",
+      "orm": "Raw",
+      "platform": "API-Hour",
+      "webserver": "Gunicorn",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "API-Hour+AsyncIO",
+      "notes": "Python 3 + API-Hour + AsyncIO"
     }
   }]
-}
+}

frameworks/Python/API-Hour/hello/etc/hello/api_hour/gunicorn_conf.py

@@ -7,7 +7,7 @@ workers = multiprocessing.cpu_count() * 2
 if _is_travis:
     workers = 2
 
-bind = "0.0.0.0:8008"
+bind = ['0.0.0.0:8008', '0.0.0.0:8009', '0.0.0.0:8011']
 keepalive = 120
 errorlog = '-'
 pidfile = 'api_hour.pid'

frameworks/Python/API-Hour/hello/etc/hello/main/main.yaml

@@ -7,4 +7,8 @@ engines:
     user: benchmarkdbuser
     password: benchmarkdbpass
     minsize: 22
-    maxsize: 22
+    maxsize: 22
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    poolsize: 40

frameworks/Python/API-Hour/hello/hello/__init__.py

@@ -5,12 +5,14 @@ import os
 import aiopg
 import jinja2
 import psycopg2.extras
+import asyncio_redis
+from asyncio_redis.protocol import HiRedisProtocol
 import aiohttp.web
 import aiohttp_jinja2
 import api_hour
 
 from . import endpoints
-
+from . import servers
 
 LOG = logging.getLogger(__name__)
 
@@ -25,9 +27,13 @@ class Container(api_hour.Container):
         # routes
         self.servers['http'].router.add_route('GET', '/json', endpoints.world.json)
         self.servers['http'].router.add_route('GET', '/db', endpoints.world.db)
+        self.servers['http'].router.add_route('GET', '/db_redis', endpoints.world.db_redis)
         self.servers['http'].router.add_route('GET', '/queries', endpoints.world.queries)
+        self.servers['http'].router.add_route('GET', '/queries_redis', endpoints.world.queries_redis)
         self.servers['http'].router.add_route('GET', '/fortunes', endpoints.world.fortunes)
+        self.servers['http'].router.add_route('GET', '/fortunes_redis', endpoints.world.fortunes_redis)
         self.servers['http'].router.add_route('GET', '/updates', endpoints.world.updates)
+        self.servers['http'].router.add_route('GET', '/updates_redis', endpoints.world.updates_redis)
         self.servers['http'].router.add_route('GET', '/plaintext', endpoints.world.plaintext)
 
     def make_servers(self):
@@ -35,8 +41,9 @@ class Container(api_hour.Container):
                                                   debug=self.worker.cfg.debug,
                                                   keep_alive=self.worker.cfg.keepalive,
                                                   access_log=self.worker.log.access_log,
-                                                  access_log_format=self.worker.cfg.access_log_format)]
-
+                                                  access_log_format=self.worker.cfg.access_log_format),
+                servers.yocto_http.YoctoHttpJson,
+                servers.yocto_http.YoctoHttpText]
 
     @asyncio.coroutine
     def start(self):
@@ -50,8 +57,14 @@ class Container(api_hour.Container):
                                                                      password=self.config['engines']['pg']['password'],
                                                                      cursor_factory=psycopg2.extras.RealDictCursor,
                                                                      minsize=int(self.config['engines']['pg']['minsize']),
-                                                                     maxsize=int(self.config['engines']['pg']['maxsize'])))
+                                                                     maxsize=int(self.config['engines']['pg']['maxsize']),
+                                                                     loop=self.loop))
         yield from asyncio.wait([self.engines['pg']], return_when=asyncio.ALL_COMPLETED)
+        self.engines['redis'] = yield from asyncio_redis.Pool.create(host=self.config['engines']['redis']['host'],
+                                                                     port=self.config['engines']['redis']['port'],
+                                                                     poolsize=self.config['engines']['redis']['poolsize'],
+                                                                     loop=self.loop,
+                                                                     protocol_class=HiRedisProtocol)
 
         LOG.info('All engines ready !')
 
@@ -65,5 +78,8 @@ class Container(api_hour.Container):
                 yield from self.engines['pg'].result().wait_closed()
             else:
                 yield from self.engines['pg'].cancel()
+        if 'redis' in self.engines:
+            self.engines['redis'].close()
+            yield from asyncio.sleep(1) # wait redis close connection
         LOG.info('All engines stopped !')
         yield from super().stop()

frameworks/Python/API-Hour/hello/hello/endpoints/world.py

@@ -7,6 +7,7 @@ import aiohttp_jinja2
 
 from ..services import queries_number
 from ..services.world import get_random_record, get_random_records, update_random_records, get_fortunes
+from ..services import redis
 
 LOG = logging.getLogger(__name__)
 
@@ -22,6 +23,13 @@ def db(request):
 
     return JSON((yield from get_random_record(container)))
 
[email protected]
+def db_redis(request):
+    """Test type 2: Single database query"""
+    container = request.app.ah_container
+
+    return JSON((yield from redis.get_random_record(container)))
+
 @asyncio.coroutine
 def queries(request):
     """Test type 3: Multiple database queries"""
@@ -30,6 +38,14 @@ def queries(request):
 
     return JSON((yield from get_random_records(container, limit)))
 
[email protected]
+def queries_redis(request):
+    """Test type 3: Multiple database queries"""
+    container = request.app.ah_container
+    limit = queries_number(request.GET.get('queries', 1))
+
+    return JSON((yield from redis.get_random_records(container, limit)))
+
 @asyncio.coroutine
 def fortunes(request):
     """Test type 4: Fortunes"""
@@ -39,6 +55,15 @@ def fortunes(request):
                                           request,
                                           {'fortunes': (yield from get_fortunes(container))})
 
[email protected]
+def fortunes_redis(request):
+    """Test type 4: Fortunes"""
+    container = request.app.ah_container
+
+    return aiohttp_jinja2.render_template('fortunes.html.j2',
+                                          request,
+                                          {'fortunes': (yield from redis.get_fortunes(container))})
+
 @asyncio.coroutine
 def updates(request):
     """Test type 5: Database updates"""
@@ -47,6 +72,14 @@ def updates(request):
 
     return JSON((yield from update_random_records(container, limit)))
 
[email protected]
+def updates_redis(request):
+    """Test type 5: Database updates"""
+    container = request.app.ah_container
+    limit = queries_number(request.GET.get('queries', 1))
+
+    return JSON((yield from redis.update_random_records(container, limit)))
+
 @asyncio.coroutine
 def plaintext(request):
     """Test type 6: Plaintext"""

frameworks/Python/API-Hour/hello/hello/servers/__init__.py

@@ -0,0 +1 @@
+from . import yocto_http

frameworks/Python/API-Hour/hello/hello/servers/yocto_http.py

@@ -0,0 +1,23 @@
+import asyncio
+
+import ujson
+
+from ..utils import generate_http_response
+
+class YoctoHttpJson(asyncio.Protocol):
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def data_received(self, data):
+        # self.transport.write(data)
+        payload = ujson.dumps({'message': 'Hello, World!'})
+        self.transport.write(generate_http_response(payload))
+
+class YoctoHttpText(asyncio.Protocol):
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def data_received(self, data):
+        # self.transport.write(data)
+        payload = 'Hello, World!'
+        self.transport.write(generate_http_response(payload, 'text/plain; charset=UTF-8'))

frameworks/Python/API-Hour/hello/hello/services/redis.py

@@ -0,0 +1,45 @@
+from operator import itemgetter
+import asyncio
+from random import randint
+
+
[email protected]
+def get_random_record(container):
+    idx = randint(1, 10000)
+    random_number = yield from container.engines['redis'].get('world:%i' % idx)
+    return {'Id': idx, 'RandomNumber': random_number}
+
[email protected]
+def get_random_records(container, limit):
+    results = []
+    for i in range(limit):
+        idx = randint(1, 10000)
+        random_number = yield from container.engines['redis'].get('world:%i' % idx)
+        results.append({'Id': idx, 'RandomNumber': random_number})
+
+    return results
+
[email protected]
+def update_random_records(container, limit):
+    results = []
+    for i in range(limit):
+        idx = randint(1, 10000)
+        random_number = yield from container.engines['redis'].get('world:%i' % idx)
+        yield from container.engines['redis'].set('world:%i' % idx, str(randint(1, 10000)))
+        results.append({'Id': idx, 'RandomNumber': random_number})
+    return results
+
[email protected]
+def get_fortunes(container):
+    results = []
+    list_reply = yield from container.engines['redis'].lrange('fortunes')
+    fortunes = yield from list_reply.aslist()
+    i = 1
+    for fortune in fortunes:
+        results.append({'id': i, 'message': fortune})
+        i += 1
+
+    results.append({'id': 0, 'message': 'Additional fortune added at request time.'})
+    results.sort(key=itemgetter('message'))
+
+    return results

frameworks/Python/API-Hour/hello/hello/utils/__init__.py

@@ -1,3 +1,14 @@
 import logging
+from wsgiref.handlers import format_date_time
 
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger(__name__)
+
+def generate_http_response(payload, content_type='application/json'):
+    return ("""HTTP/1.1 200 OK
+CONTENT-TYPE: %s
+CONTENT-LENGTH: %s
+CONNECTION: keep-alive
+DATE: %s
+SERVER: yocto_http/0.0.1
+
+%s""" % (content_type, len(payload), format_date_time(None), payload)).encode('utf-8')

frameworks/Python/API-Hour/requirements.txt

@@ -1,11 +1,14 @@
-api_hour==0.6.1
+Jinja2==2.7.3
+MarkupSafe==0.23
+PyYAML==3.11
+aiohttp==0.13.1
+aiohttp-jinja2==0.0.2
 aiopg==0.5.2
+api-hour==0.6.1
+asyncio-redis==0.13.4
 gunicorn==19.1.1
+hiredis==0.1.6
 psycopg2==2.5.4
 setproctitle==1.1.8
 six==1.8.0
 ujson==1.33
-aiohttp==0.13.1
-PyYAML==3.11
-aiohttp-jinja2==0.0.2
-MarkupSafe==0.23