Merge pull request #1989 from methane/python-update

Python update for Round 13

.travis.yml

@@ -219,6 +219,7 @@ addons:
       - redis-server
 
 before_script:
+  - sudo sysctl -w net.core.somaxconn=65535
   - sudo ./config/travis_mysql_setup.sh
   - mysql -uroot < config/create.sql
   - sudo ./config/create-redis.sh

frameworks/Python/.gitignore

@@ -1,6 +1,7 @@
 uwsgi_params
 *.pid
+__pycache__
 
 # ignore virtual environments
 pyvenv*/
-venv*/
+venv*/

frameworks/Python/README.md

@@ -6,10 +6,11 @@ For further guidance, review the
 
 ## Infrastructure Software Versions
 
-* [Python2 2.7.10](https://www.python.org/)
-* [Python3 3.4.3](https://www.python.org/)
-* [PyPy 2.6.0](http://pypy.org/)
+* [CPython2](https://www.python.org/)
+* [CPython3](https://www.python.org/)
+* [PyPy](http://pypy.org/)
 * [nginx](http://nginx.org/)
+* [uWSGI](https://uwsgi-docs.readthedocs.org/en/latest/)
 
 ## Get Help
 
@@ -34,9 +35,7 @@ This is a bit slower than CPython 2, but more simple and productive than it.
 
 ### (C)Python 2
 
-Legacy Python.
-
-It is most major interpreter for now.
+Legacy Python.  Still used for widely.
 
 ### PyPy
 
@@ -45,6 +44,7 @@ PyPy is the fastest Python implementation with JIT.
 There is PyPy's Python 3 implementation (PyPy3), but it is not so tuned like PyPy2.
 So we don't use it for now.
 
+
 ## WSGI Servers
 
 ### Nginx + uWSGI
@@ -85,6 +85,7 @@ We can't maintain matrix of frameworks (5~) * servers (3~) * DBs (3) * Interpret
 
 If you want to know about performance difference between MySQL and PostgreSQL,
 there are no need to all frameworks implement both of them.
+
 Additionally, Python is not a good language to measure performance of DBs because
 web application written in Python is slower than DBs.
 

frameworks/Python/bottle/requirements-pypy.txt

@@ -1,6 +1,6 @@
-bottle==0.12.8
-bottle-sqlalchemy==0.4.2
-SQLAlchemy==1.0.4
-gunicorn==19.3.0
-PyMySQL==0.6.7
-tornado==4.2
+bottle==0.12.9
+bottle-sqlalchemy==0.4.3
+SQLAlchemy==1.0.12
+gunicorn==19.4.5
+PyMySQL==0.7.2
+tornado==4.3

frameworks/Python/bottle/requirements.txt

@@ -1,8 +1,8 @@
-bottle==0.12.8
-bottle-sqlalchemy==0.4.2
-SQLAlchemy==1.0.4
-gunicorn==19.3.0
-meinheld==0.5.7
-mysqlclient==1.3.6
-greenlet==0.4.7
-uwsgi
+bottle==0.12.9
+bottle-sqlalchemy==0.4.3
+SQLAlchemy==1.0.12
+gunicorn==19.4.5
+meinheld==0.5.9
+mysqlclient==1.3.7
+greenlet==0.4.9
+uwsgi==2.0.12

frameworks/Python/cherrypy/requirements.txt

@@ -1,3 +1,3 @@
-cherrypy==3.7.0
-SQLAlchemy==1.0.4
-mysqlclient==1.3.6
+cherrypy==5.1.0
+SQLAlchemy==1.0.12
+mysqlclient==1.3.7

frameworks/Python/django/requirements.txt

@@ -1,10 +1,10 @@
-Django==1.8.2
+Django==1.9.4
 
-mysqlclient==1.3.6
-psycopg2==2.6
-ujson==1.33
+mysqlclient==1.3.7
+psycopg2==2.6.1
+ujson==1.35
 
-gunicorn==19.3.0
-meinheld==0.5.7
+gunicorn==19.4.5
+meinheld==0.5.9
 
-greenlet==0.4.7
+greenlet==0.4.9

frameworks/Python/falcon/requirements-pypy.txt

@@ -1,3 +1,3 @@
-gunicorn==19.3.0
-tornado==4.2
+gunicorn==19.4.5
+tornado==4.3
 falcon==0.3.0

frameworks/Python/falcon/requirements.txt

@@ -1,6 +1,5 @@
-gunicorn==19.3.0
-meinheld==0.5.7
-Cython==0.22
+gunicorn==19.4.5
+meinheld==0.5.9
+Cython==0.23.4
 falcon==0.3.0
-
-greenlet==0.4.7
+greenlet==0.4.9

frameworks/Python/flask/requirements-pypy.txt

@@ -1,8 +1,8 @@
-Jinja2==2.7.3
-Werkzeug==0.10.4
+Jinja2==2.8
+Werkzeug==0.11.4
 flask==0.10.1
-SQLAlchemy==1.0.4
-Flask-SQLAlchemy==2.0
-PyMySQL==0.6.7
-gunicorn==19.3.0
-tornado==4.2
+SQLAlchemy==1.0.12
+Flask-SQLAlchemy==2.1
+PyMySQL==0.7.2
+gunicorn==19.4.5
+tornado==4.3

frameworks/Python/flask/requirements.txt

@@ -1,11 +1,10 @@
-Jinja2==2.7.3
-Werkzeug==0.10.4
+Jinja2==2.8
+Werkzeug==0.11.4
 flask==0.10.1
-SQLAlchemy==1.0.11
-Flask-SQLAlchemy==2.0
-mysqlclient==1.3.6
-gunicorn==19.3.0
-meinheld==0.5.7
+SQLAlchemy==1.0.12
+Flask-SQLAlchemy==2.1
+mysqlclient==1.3.7
+gunicorn==19.4.5
+meinheld==0.5.9
 uwsgi==2.0.12
-
-greenlet==0.4.7
+greenlet==0.4.9

frameworks/Python/klein/requirements.txt

@@ -1,5 +1,4 @@
-klein==15.0.0
-
-mysqlclient==1.3.6
-SQLAlchemy==1.0.4
-jinja2==2.7.3
+klein==15.3.1
+mysqlclient==1.3.7
+SQLAlchemy==1.0.12
+jinja2==2.8

frameworks/Python/pyramid/requirements.txt

@@ -1,8 +1,7 @@
-psycopg2==2.6
-gunicorn==19.3.0
-meinheld==0.5.7
-SQLAlchemy==1.0.4
-pyramid==1.5.7
+psycopg2==2.6.1
+gunicorn==19.4.5
+meinheld==0.5.9
+SQLAlchemy==1.0.12
+pyramid==1.6.1
 pyramid_chameleon==0.3
-
-greenlet==0.4.7
+greenlet==0.4.9

frameworks/Python/tornado/requirements.txt

@@ -1,3 +1,3 @@
-tornado==4.2
-motor==0.4.1
-Momoko==2.0.0
+tornado==4.3
+motor==0.6.2
+Momoko==2.2.3

frameworks/Python/turbogears/requirements.txt

@@ -1,9 +1,10 @@
-tg.devtools==2.3.5
+tg.devtools==2.3.8
 
-SQLAlchemy==1.0.4
+SQLAlchemy==1.0.12
 zope.sqlalchemy==0.7.6
-mysqlclient==1.3.6
-jinja2==2.7.3
+mysqlclient==1.3.7
+jinja2==2.8
 
-gunicorn==19.3.0
-meinheld==0.5.7
+gunicorn==19.4.5
+meinheld==0.5.9
+greenlet==0.4.9

frameworks/Python/uwsgi/requirements.txt

@@ -1,3 +1,3 @@
-gevent==1.0.2
-uwsgi
-ujson==1.33
+gevent==1.1.0
+uwsgi==2.0.12
+ujson==1.35

frameworks/Python/wheezyweb/benchmark_config.json

@@ -20,8 +20,8 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "wheezy.web",
-      "notes": "CPython 2.7"
+      "display_name": "wheezy.web-py2",
+      "notes": ""
     },
     "py3": {
       "setup_file": "setup_py3",
@@ -43,7 +43,7 @@
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "wheezy.web-py3",
-      "notes": "CPython 3.4"
+      "notes": ""
     }
   }]
 }

frameworks/Python/wheezyweb/requirements.txt

@@ -1,10 +1,10 @@
 wheezy.web==0.1.485
 wheezy.template==0.1.167
 
-SQLAlchemy==1.0.4
-mysqlclient==1.3.6
+SQLAlchemy==1.0.12
+mysqlclient==1.3.7
 
-gunicorn==19.3.0
-meinheld==0.5.7
+gunicorn==19.4.5
+meinheld==0.5.9
 
-greenlet==0.4.7
+greenlet==0.4.9

frameworks/Python/wsgi/benchmark_config.json

@@ -16,26 +16,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "wsgi-meinheld",
-      "notes": "",
-      "versus": "wsgi"
-    },
-    "nginx-uwsgi": {
-      "setup_file": "setup_nginxuwsgi",
-      "json_url": "/json",
-      "plaintext_url": "/plaintext",
-      "port": 8080,
-      "approach": "Stripped",
-      "classification": "Platform",
-      "database": "None",
-      "framework": "wsgi",
-      "language": "Python",
-      "orm": "Raw",
-      "platform": "uWSGI",
-      "webserver": "nginx",
-      "os": "Linux",
-      "database_os": "Linux",
-      "display_name": "wsgi-nginx-uWSGI",
+      "display_name": "meinheld",
       "notes": "",
       "versus": "wsgi"
     }

frameworks/Python/wsgi/hello.py

@@ -15,7 +15,7 @@ def json(environ, start_response):
         ('Content-type', 'application/json'),
         ('Content-Length', str(len(data)))
     ]
-    start_response(b'200 OK', response_headers)
+    start_response('200 OK', response_headers)
     return [data]
 
 def plaintext(environ, start_response):
@@ -24,7 +24,7 @@ def plaintext(environ, start_response):
         ('Content-type', 'text/plain'),
         ('Content-Length', str(len(data)))
     ]
-    start_response(b'200 OK', response_headers)
+    start_response('200 OK', response_headers)
     return [data]
 
 def app(environ, start_response):

frameworks/Python/wsgi/nginx.conf

@@ -1,48 +0,0 @@
-# This file is based on /usr/local/nginx/conf/nginx.conf.default.
-
-# One worker process per core
-error_log stderr error;
-
-events {
-    # This needed to be increased because the nginx error log said so.
-    # http://nginx.org/en/docs/ngx_core_module.html#worker_connections
-    worker_connections  65535;
-    multi_accept on;
-}
-
-http {
-    default_type  application/octet-stream;
-    client_body_temp_path      /tmp;
-
-    # turn off request logging for performance
-    access_log off;
-
-    # I think these only options affect static file serving
-    sendfile        on;
-    tcp_nopush      on;
-
-    # Allow many HTTP Keep-Alive requests in a single TCP connection before
-    # closing it (the default is 100). This will minimize the total number
-    # of TCP connections opened/closed. The problem is that this may cause
-    # some worker processes to be handling too connections relative to the
-    # other workers based on an initial imbalance, so this is disabled for
-    # now.
-#    keepalive_requests 1000;
-
-    #keepalive_timeout  0;
-    keepalive_timeout  65;
-
-    server {
-        # For information on deferred, see:
-        # http://nginx.org/en/docs/http/ngx_http_core_module.html#listen
-        # http://www.techrepublic.com/article/take-advantage-of-tcp-ip-options-to-optimize-data-transmission/
-        # The backlog argument to listen() is set to match net.ipv4.tcp_max_syn_backlog and net.core.somaxconn
-        listen       8080 default_server deferred backlog=65535;
-        server_name  localhost;
-
-        location / {
-            uwsgi_pass unix:/tmp/uwsgi.sock;
-            include /usr/local/nginx/conf/uwsgi_params;
-        }
-    }    
-}

frameworks/Python/wsgi/requirements.txt

@@ -1,6 +1,4 @@
-ujson==1.33
-uwsgi
-gunicorn==19.3.0
-meinheld==0.5.7
-
-greenlet==0.4.7
+ujson==1.35
+gunicorn==19.4.5
+meinheld==0.5.9
+greenlet==0.4.9

frameworks/Python/wsgi/setup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-fw_depends python2
+fw_depends python3
 
-pip install --install-option="--prefix=${PY2_ROOT}" -r $TROOT/requirements.txt
+pip install --install-option="--prefix=${PY3_ROOT}" -r $TROOT/requirements.txt
 
 gunicorn hello:app -c gunicorn_conf.py &

frameworks/Python/wsgi/setup_nginxuwsgi.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-fw_depends python2 nginx
-
-sed -i 's|include .*/conf/uwsgi_params;|include '"${NGINX_HOME}"'/conf/uwsgi_params;|g' nginx.conf
-
-pip install --install-option="--prefix=${PY2_ROOT}" -r $TROOT/requirements.txt
-
-nginx -c $TROOT/nginx.conf
-uwsgi --ini uwsgi.ini --processes $MAX_THREADS --wsgi hello:app &

frameworks/Python/wsgi/uwsgi.ini

@@ -1,19 +0,0 @@
-[uwsgi]
-master
-; Increase listen queue used for nginx connecting to uWSGI. This matches
-; net.ipv4.tcp_max_syn_backlog and net.core.somaxconn.
-listen = 65535
-; for performance
-disable-logging
-; use UNIX sockets instead of TCP loopback for performance
-socket = /tmp/uwsgi.sock
-; allow nginx to access the UNIX socket
-chmod-socket = 666
-; Avoid thundering herd problem http://uwsgi-docs.readthedocs.org/en/latest/articles/SerializingAccept.html .
-; This is currently disabled because when I tried it with flask, it caused a
-; 20% performance hit. The CPU cores could not be saturated with thunder-lock.
-; I'm not yet sure the full story, so this is presently disabled. Also,
-; disabling this caused bottle to get ~13% faster.
-;thunder-lock
-; used by uwsgi_stop.ini
-pidfile = /tmp/uwsgi.pid

toolset/setup/linux/languages/pypy.sh

@@ -6,7 +6,7 @@ RETCODE=$(fw_exists ${IROOT}/pypy.installed)
   return 0; }
   
 PYPY_ROOT=$IROOT/pypy
-PYPY_VERSION=5.0.0
+PYPY_VERSION=5.0.1
 
 fw_get -o pypy-${PYPY_VERSION}-linux64.tar.bz2 https://bitbucket.org/pypy/pypy/downloads/pypy-${PYPY_VERSION}-linux64.tar.bz2
 fw_untar pypy-${PYPY_VERSION}-linux64.tar.bz2

toolset/setup/linux/languages/python2.sh

@@ -6,17 +6,18 @@ RETCODE=$(fw_exists ${IROOT}/py2.installed)
   return 0; }
 
 PY2_ROOT=$IROOT/py2
+PY2_VERSION=2.7.11
 
-fw_get -O http://www.python.org/ftp/python/2.7.10/Python-2.7.10.tgz
-fw_untar Python-2.7.10.tgz
-cd Python-2.7.10
+fw_get -O http://www.python.org/ftp/python/${PY2_VERSION}/Python-${PY2_VERSION}.tar.xz
+fw_untar Python-${PY2_VERSION}.tar.xz
+cd Python-${PY2_VERSION}
 ./configure --prefix=${IROOT}/py2 --disable-shared --quiet
 make -j4 --quiet 2>&1 | tee $IROOT/python-install.log | awk '{ if (NR%100 == 0) printf "."}'
 make install --quiet 2>&1 | tee -a $IROOT/python-install.log | awk '{ if (NR%100 == 0) printf "."}'
 cd ..
 
 $PY2_ROOT/bin/python -m ensurepip -U
-$PY2_ROOT/bin/pip install -U setuptools pip
+$PY2_ROOT/bin/pip install -U setuptools pip wheel
 
 echo "export PY2_ROOT=${PY2_ROOT}" > $IROOT/py2.installed
 echo -e "export PYTHONHOME=\$PY2_ROOT" >> $IROOT/py2.installed

toolset/setup/linux/languages/python3.sh

@@ -6,17 +6,18 @@ RETCODE=$(fw_exists ${IROOT}/py3.installed)
   return 0; }
   
 PY3_ROOT=$IROOT/py3
+PY3_VERSION=3.5.1
 
-fw_get -O http://www.python.org/ftp/python/3.5.1/Python-3.5.1.tar.xz
-fw_untar Python-3.5.1.tar.xz
-cd Python-3.5.1
+fw_get -O http://www.python.org/ftp/python/${PY3_VERSION}/Python-${PY3_VERSION}.tar.xz
+fw_untar Python-${PY3_VERSION}.tar.xz
+cd Python-${PY3_VERSION}
 ./configure --prefix=$PY3_ROOT --disable-shared --with-computed-gotos --quiet
 make -j4 --quiet 2>&1 | tee $IROOT/python3-install.log | awk '{ if (NR%100 == 0) printf "."}'
 make install --quiet 2>&1 | tee -a $IROOT/python3-install.log | awk '{ if (NR%100 == 0) printf "."}'
 cd ..
 
 $PY3_ROOT/bin/python3 -m ensurepip -U
-$PY3_ROOT/bin/pip3 install -U setuptools pip
+$PY3_ROOT/bin/pip3 install -U setuptools pip wheel
 
 echo "export PY3_ROOT=${PY3_ROOT}" > $IROOT/py3.installed
 echo -e "export PYTHONHOME=\$PY3_ROOT" >> $IROOT/py3.installed