installer.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669
  1. import subprocess
  2. import os
  3. import time
  4. import traceback
  5. import sys
  6. class Installer:
  7. ############################################################
  8. # install_software
  9. ############################################################
  10. def install_software(self):
  11. if self.benchmarker.install == 'all' or self.benchmarker.install == 'server':
  12. self.__install_server_software()
  13. if self.benchmarker.install == 'all' or self.benchmarker.install == 'database':
  14. self.__install_database_software()
  15. if self.benchmarker.install == 'all' or self.benchmarker.install == 'client':
  16. self.__install_client_software()
  17. ############################################################
  18. # End install_software
  19. ############################################################
  20. ############################################################
  21. # __install_server_software
  22. ############################################################
  23. def __install_server_software(self):
  24. print("\nINSTALL: Installing server software\n")
  25. #######################################
  26. # Prerequisites
  27. #######################################
  28. self.__run_command("sudo apt-get update", True)
  29. self.__run_command("sudo apt-get upgrade", True)
  30. self.__run_command("sudo apt-get install build-essential libpcre3 libpcre3-dev libpcrecpp0 libssl-dev zlib1g-dev python-software-properties unzip git-core libcurl4-openssl-dev libbz2-dev libmysqlclient-dev mongodb-clients libreadline6-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt-dev libgdbm-dev ncurses-dev automake libffi-dev htop libtool bison libevent-dev libgstreamer-plugins-base0.10-0 libgstreamer0.10-0 liborc-0.4-0 libwxbase2.8-0 libwxgtk2.8-0 libgnutls-dev libjson0-dev libmcrypt-dev libicu-dev cmake gettext curl libpq-dev mercurial mlton", True)
  31. self.__run_command("sudo add-apt-repository ppa:ubuntu-toolchain-r/test", True)
  32. self.__run_command("sudo apt-get update", True)
  33. self.__run_command("sudo apt-get install gcc-4.8 g++-4.8", True)
  34. self.__run_command("cp ../config/benchmark_profile ../../.bash_profile")
  35. self.__run_command("cat ../config/benchmark_profile >> ../../.profile")
  36. self.__run_command("cat ../config/benchmark_profile >> ../../.bashrc")
  37. self.__run_command(". ../../.profile")
  38. self.__run_command("sudo sh -c \"echo '* - nofile 65535' >> /etc/security/limits.conf\"")
  39. ##############################################################
  40. # System Tools
  41. ##############################################################
  42. #
  43. # Leiningen
  44. #
  45. self.__run_command("mkdir -p bin")
  46. self.__download("https://raw.github.com/technomancy/leiningen/stable/bin/lein")
  47. self.__run_command("mv lein bin/lein")
  48. self.__run_command("chmod +x bin/lein")
  49. #
  50. # Maven
  51. #
  52. self.__run_command("sudo apt-get install maven -qq")
  53. self.__run_command("mvn -version")
  54. #######################################
  55. # Languages
  56. #######################################
  57. self._install_python()
  58. #
  59. # Dart
  60. #
  61. self.__download("http://storage.googleapis.com/dart-archive/channels/stable/release/latest/sdk/dartsdk-linux-x64-release.zip")
  62. self.__run_command("unzip -qqo dartsdk-linux-x64-release.zip")
  63. #
  64. # Erlang
  65. #
  66. self.__run_command("sudo cp ../config/erlang.list /etc/apt/sources.list.d/erlang.list")
  67. self.__download("http://binaries.erlang-solutions.com/debian/erlang_solutions.asc")
  68. self.__run_command("sudo apt-key add erlang_solutions.asc")
  69. self.__run_command("sudo apt-get update")
  70. self.__run_command("sudo apt-get install esl-erlang", True)
  71. #
  72. # nodejs
  73. #
  74. self.__download("http://nodejs.org/dist/v0.10.8/node-v0.10.8-linux-x64.tar.gz")
  75. self.__run_command("tar xzf node-v0.10.8-linux-x64.tar.gz")
  76. #
  77. # Java
  78. #
  79. self.__run_command("sudo apt-get install openjdk-7-jdk", True)
  80. self.__run_command("sudo apt-get remove --purge openjdk-6-jre openjdk-6-jre-headless", True)
  81. #
  82. # Elixir
  83. #
  84. self.__run_command("wget https://github.com/elixir-lang/elixir/archive/v0.13.3.tar.gz");
  85. self.__run_command("sudo tar -zxf v0.13.3.tar.gz");
  86. self.__run_command("sudo make clean", cwd="elixir-0.13.3");
  87. self.__run_command("sudo make test", cwd="elixir-0.13.3");
  88. #
  89. # Ruby/JRuby
  90. #
  91. self.__run_command("curl -L get.rvm.io | bash -s head --auto-dotfiles")
  92. self.__run_command("echo rvm_auto_reload_flag=2 >> ~/.rvmrc")
  93. self.__bash_from_string("source ~/.rvm/scripts/'rvm' && rvm install 2.0.0-p0")
  94. self.__bash_from_string("source ~/.rvm/scripts/'rvm' && rvm 2.0.0-p0 do gem install bundler")
  95. self.__bash_from_string("source ~/.rvm/scripts/'rvm' && rvm install jruby-1.7.8")
  96. self.__bash_from_string("source ~/.rvm/scripts/'rvm' && rvm jruby-1.7.8 do gem install bundler")
  97. #
  98. # go
  99. #
  100. self.__download("https://storage.googleapis.com/golang/go1.3.linux-amd64.tar.gz");
  101. self.__run_command("tar xzf go1.3.linux-amd64.tar.gz")
  102. #
  103. # Perl
  104. #
  105. self.__download("https://raw.githubusercontent.com/tokuhirom/Perl-Build/master/perl-build", "perl-build.pl")
  106. self.__run_command("perl perl-build.pl -DDEBUGGING=-g 5.18.2 ~/FrameworkBenchmarks/installs/perl-5.18", retry=True)
  107. self.__download("http://cpanmin.us", "cpanminus.pl")
  108. self.__run_command("~/FrameworkBenchmarks/installs/perl-5.18/bin/perl cpanminus.pl --notest --no-man-page App::cpanminus", retry=True)
  109. self.__run_command("~/FrameworkBenchmarks/installs/perl-5.18/bin/cpanm -f --notest --no-man-page DBI DBD::mysql Kelp Dancer Mojolicious Kelp::Module::JSON::XS Dancer::Plugin::Database Starman Plack JSON Web::Simple DBD::Pg JSON::XS EV HTTP::Parser::XS Monoceros EV IO::Socket::IP IO::Socket::SSL Memoize", retry=True)
  110. #
  111. # php
  112. #
  113. self.__download("http://museum.php.net/php5/php-5.4.13.tar.gz")
  114. self.__run_command("tar xzf php-5.4.13.tar.gz")
  115. self.__run_command("./configure --with-pdo-mysql --with-mysql --with-mcrypt --enable-intl --enable-mbstring --enable-fpm --with-fpm-user=www-data --with-fpm-group=www-data --with-openssl", cwd="php-5.4.13")
  116. self.__run_command("make", cwd="php-5.4.13")
  117. self.__run_command("sudo make install", cwd="php-5.4.13")
  118. self.__run_command("printf \"\\n\" | sudo pecl install -f apc-beta", cwd="php-5.4.13", retry=True)
  119. self.__run_command("sudo cp ../config/php.ini /usr/local/lib/php.ini")
  120. self.__run_command("sudo cp ../config/php-fpm.conf /usr/local/lib/php-fpm.conf")
  121. # Composer
  122. self.__download("https://getcomposer.org/installer", "composer-installer.php")
  123. self.__run_command("php composer-installer.php --install-dir=bin")
  124. # Phalcon
  125. self.__run_command("git clone git://github.com/phalcon/cphalcon.git", retry=True)
  126. self.__run_command("sudo ./install", cwd="cphalcon/build")
  127. # YAF
  128. self.__run_command("sudo pecl install -f yaf")
  129. #
  130. # Haskell
  131. #
  132. self.__run_command("sudo apt-get install ghc cabal-install", True)
  133. #
  134. # RingoJs
  135. #
  136. self.__download("http://www.ringojs.org/downloads/ringojs_0.10-1_all.deb")
  137. self.__run_command("sudo apt-get install jsvc", True)
  138. self.__run_command("sudo dpkg -i ringojs_0.10-1_all.deb", True)
  139. self.__run_command("rm ringojs_0.10-1_all.deb")
  140. #
  141. # Mono
  142. #
  143. self.__download("http://download.mono-project.com/sources/mono/mono-3.2.8.tar.bz2", "mono-3.2.8.tar.bz2")
  144. self.__run_command("tar xf mono-3.2.8.tar.bz2")
  145. self.__run_command("./configure --disable-nls --prefix=/usr/local", cwd="mono-3.2.8")
  146. self.__run_command("make get-monolite-latest", cwd="mono-3.2.8")
  147. self.__run_command("make -j4 EXTERNAL_MCS=${PWD}/mcs/class/lib/monolite/basic.exe", cwd="mono-3.2.8")
  148. self.__run_command("sudo make install", cwd="mono-3.2.8")
  149. self.__run_command("mv mono-3.2.8 mono");
  150. self.__run_command("mozroots --import --sync", retry=True)
  151. self.__run_command("git clone --depth 1 git://github.com/mono/xsp", retry=True)
  152. self.__run_command("./autogen.sh --prefix=/usr/local", cwd="xsp")
  153. self.__run_command("make", cwd="xsp")
  154. self.__run_command("sudo make install", cwd="xsp")
  155. #
  156. # Nimrod
  157. #
  158. self.__run_command("git clone git://github.com/Araq/Nimrod.git nimrod", retry=True)
  159. self.__run_command("git checkout 987ac2439a87d74838233a7b188e4db340495ee5", cwd="nimrod")
  160. self.__run_command("git clone git://github.com/nimrod-code/csources.git", cwd="nimrod", retry=True)
  161. self.__run_command("git checkout 704015887981932c78a033dd5ede623b2ad6ae27", cwd="nimrod/csources")
  162. self.__run_command("chmod +x build.sh", cwd="nimrod/csources")
  163. self.__run_command("./build.sh", cwd="nimrod/csources")
  164. self.__run_command("bin/nimrod c koch", cwd="nimrod")
  165. self.__run_command("./koch boot -d:release", cwd="nimrod")
  166. #
  167. # Racket
  168. #
  169. self.__download("https://github.com/plt/racket/archive/v5.3.6.tar.gz", "racket-5.3.6.tar.gz")
  170. self.__run_command("tar xzf racket-5.3.6.tar.gz")
  171. self.__run_command("./configure", cwd="racket-5.3.6/src")
  172. self.__run_command("make", cwd="racket-5.3.6/src")
  173. self.__run_command("sudo make install", cwd="racket-5.3.6/src")
  174. #
  175. # Ur/Web
  176. #
  177. self.__run_command("hg clone http://hg.impredicative.com/urweb")
  178. self.__run_command("./autogen.sh", cwd="urweb")
  179. self.__run_command("./configure", cwd="urweb")
  180. self.__run_command("make", cwd="urweb")
  181. self.__run_command("sudo make install", cwd="urweb")
  182. #
  183. # HHVM
  184. #
  185. self.__run_command("wget -O - http://dl.hhvm.com/conf/hhvm.gpg.key | sudo apt-key add -")
  186. self.__run_command("echo deb http://dl.hhvm.com/ubuntu trusty main | sudo tee /etc/apt/sources.list.d/hhvm.list")
  187. self.__run_command("sudo apt-get update")
  188. self.__run_command("sudo apt-get install hhvm", True)
  189. #######################################
  190. # Webservers
  191. #######################################
  192. #
  193. # Nginx
  194. #
  195. self.__download("http://nginx.org/download/nginx-1.4.1.tar.gz")
  196. self.__run_command("tar xzf nginx-1.4.1.tar.gz")
  197. self.__run_command("./configure", cwd="nginx-1.4.1")
  198. self.__run_command("make", cwd="nginx-1.4.1")
  199. self.__run_command("sudo make install", cwd="nginx-1.4.1")
  200. #
  201. # Openresty (nginx with lua stuff)
  202. #
  203. self.__download("http://openresty.org/download/ngx_openresty-1.5.8.1.tar.gz")
  204. self.__run_command("tar xzf ngx_openresty-1.5.8.1.tar.gz")
  205. self.__run_command("./configure --with-luajit --with-http_postgres_module", cwd="ngx_openresty-1.5.8.1")
  206. self.__run_command("make", cwd="ngx_openresty-1.5.8.1")
  207. self.__run_command("sudo make install", cwd="ngx_openresty-1.5.8.1")
  208. #
  209. # Lapis
  210. #
  211. self.__run_command("sudo apt-get install luarocks", True)
  212. self.__run_command("sudo luarocks install http://github.com/leafo/lapis/raw/master/lapis-dev-1.rockspec")
  213. #
  214. # Resin
  215. #
  216. self.__run_command("sudo cp -r /usr/lib/jvm/java-1.7.0-openjdk-amd64/include /usr/lib/jvm/java-1.7.0-openjdk-amd64/jre/bin/")
  217. self.__download("http://www.caucho.com/download/resin-4.0.36.tar.gz")
  218. self.__run_command("tar xzf resin-4.0.36.tar.gz")
  219. self.__run_command("./configure --prefix=`pwd`", cwd="resin-4.0.36")
  220. self.__run_command("make", cwd="resin-4.0.36")
  221. self.__run_command("make install", cwd="resin-4.0.36")
  222. self.__run_command("mv conf/resin.properties conf/resin.properties.orig", cwd="resin-4.0.36")
  223. self.__run_command("cat ../config/resin.properties > resin-4.0.36/conf/resin.properties")
  224. self.__run_command("mv conf/resin.xml conf/resin.xml.orig", cwd="resin-4.0.36")
  225. self.__run_command("cat ../config/resin.xml > resin-4.0.36/conf/resin.xml")
  226. #
  227. # Mongrel2
  228. #
  229. self.__download("http://download.zeromq.org/zeromq-4.0.3.tar.gz")
  230. self.__run_command("tar xzf zeromq-4.0.3.tar.gz")
  231. self.__run_command("./configure", cwd="zeromq-4.0.3")
  232. self.__run_command("make", cwd="zeromq-4.0.3")
  233. self.__run_command("sudo make install", cwd="zeromq-4.0.3")
  234. self.__run_command("sudo apt-get install sqlite3 libsqlite3-dev uuid uuid-runtime uuid-dev", True)
  235. self.__run_command("sudo ldconfig -v")
  236. self.__download("https://github.com/zedshaw/mongrel2/tarball/v1.8.1", "mongrel2.tar.gz")
  237. self.__run_command("tar xvf mongrel2.tar.gz")
  238. self.__run_command("mv zedshaw-mongrel2-aa2ecf8 mongrel2")
  239. # for zmq4, we update the following file manually (not in v1.8.1)
  240. self.__download("https://raw.github.com/zedshaw/mongrel2/9b565eeea003783c47502c2d350b99c9684ce97c/src/zmq_compat.h")
  241. self.__run_command("mv -f zmq_compat.h mongrel2/src/")
  242. self.__run_command("make clean all && sudo make install", cwd="mongrel2")
  243. #
  244. # Weber
  245. #
  246. self.__run_command("git clone https://github.com/elixir-web/weber.git");
  247. # To get the two make commands working, we need to hard code the path for elixir's "mix" and run it in bash
  248. self.__run_command("sed -i 's:$(MIX):/home/tfb/FrameworkBenchmarks/installs/elixir-0.13.3/bin/mix:' Makefile", cwd="weber")
  249. # Because of the way that we are running it, we can't just "send a yes" like normal
  250. self.__run_command("bash -i -c 'yes | make'", cwd="weber");
  251. self.__run_command("bash -i -c 'yes | make test'", cwd="weber");
  252. ##############################################################
  253. # Frameworks
  254. ##############################################################
  255. #
  256. # Grails
  257. #
  258. self.__download("http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/grails-2.4.2.zip")
  259. self.__run_command("unzip -o grails-2.4.2.zip")
  260. #
  261. # Play 2
  262. #
  263. self.__download("http://downloads.typesafe.com/play/2.2.0/play-2.2.0.zip")
  264. self.__run_command("unzip -o play-2.2.0.zip")
  265. #
  266. # Play 1
  267. #
  268. self.__download("http://downloads.typesafe.com/releases/play-1.2.5.zip")
  269. self.__run_command("unzip -o play-1.2.5.zip")
  270. self.__run_command("mv play-1.2.5/play play-1.2.5/play1")
  271. # siena
  272. self.__run_command("yes | play-1.2.5/play1 install siena")
  273. #
  274. # TreeFrog Framework
  275. #
  276. self.__run_command("sudo apt-get install qt4-qmake libqt4-dev libqt4-sql-mysql libqt4-sql-psql g++", True)
  277. self.__download("http://downloads.sourceforge.net/project/treefrog/src/treefrog-1.7.5.tar.gz")
  278. self.__run_command("tar xzf treefrog-1.7.5.tar.gz")
  279. self.__run_command("rm treefrog-1.7.5.tar.gz")
  280. self.__run_command("./configure", cwd="treefrog-1.7.5")
  281. self.__run_command("make", cwd="treefrog-1.7.5/src")
  282. self.__run_command("sudo make install", cwd="treefrog-1.7.5/src")
  283. self.__run_command("make", cwd="treefrog-1.7.5/tools")
  284. self.__run_command("sudo make install", cwd="treefrog-1.7.5/tools")
  285. #
  286. # Vert.x
  287. #
  288. self.__download("http://dl.bintray.com/vertx/downloads/vert.x-2.1.1.tar.gz?direct=true", "vert.x-2.1.1.tar.gz")
  289. self.__run_command("tar xzf vert.x-2.1.1.tar.gz")
  290. #
  291. # Yesod
  292. #
  293. self.__run_command("cabal update", retry=True)
  294. self.__run_command("cabal install yesod persistent-mysql", retry=True)
  295. #
  296. # Jester
  297. #
  298. self.__run_command("git clone git://github.com/dom96/jester.git jester/jester", retry=True)
  299. #
  300. # Onion
  301. #
  302. self.__run_command("git clone https://github.com/davidmoreno/onion.git")
  303. self.__run_command("mkdir build", cwd="onion")
  304. self.__run_command("cmake ..", cwd="onion/build")
  305. self.__run_command("make", cwd="onion/build")
  306. # nawak
  307. #
  308. self.__run_command("git clone git://github.com/idlewan/nawak.git nawak/nawak", retry=True)
  309. #
  310. # Wt
  311. #
  312. self.__run_command("sudo apt-get install libboost1.54-all-dev", True)
  313. self.__download("http://downloads.sourceforge.net/witty/wt-3.3.3.tar.gz", filename="wt.tar.gz")
  314. self.__run_command("tar xf wt.tar.gz")
  315. self.__run_command("rm wt.tar.gz")
  316. self.__run_command("bash -c 'mv wt-* wt'")
  317. self.__run_command("mkdir build", cwd="wt")
  318. self.__run_command("cmake .. -DWT_CPP_11_MODE=-std=c++0x -DCMAKE_BUILD_TYPE=Release", cwd="wt/build")
  319. self.__run_command("make", cwd="wt/build")
  320. self.__run_command("sudo make install", cwd="wt/build")
  321. print("\nINSTALL: Finished installing server software\n")
  322. ############################################################
  323. # End __install_server_software
  324. ############################################################
  325. def _install_python(self):
  326. """Install Python runtime, frameworks and libraries"""
  327. # PyPy 2.3.1
  328. f = "pypy-2.3.1-linux64.tar.bz2"
  329. if not os.path.exists(f):
  330. self.__download("https://bitbucket.org/pypy/pypy/downloads/" + f, f)
  331. self.__run_command("tar xf " + f)
  332. self.__run_command('ln -sf pypy-2.3.1-linux64 pypy')
  333. # CPython 2.7.7
  334. f = "Python-2.7.7.tgz"
  335. if not os.path.exists(f):
  336. self.__download("http://www.python.org/ftp/python/2.7.7/" + f, f)
  337. self.__run_command("tar xf " + f)
  338. self.__run_command("./configure --prefix=$HOME/FrameworkBenchmarks/installs/py2 --disable-shared", cwd="Python-2.7.7")
  339. self.__run_command("make -j4", cwd="Python-2.7.7")
  340. self.__run_command("make install", cwd="Python-2.7.7")
  341. # CPython 3.4.1
  342. f = "Python-3.4.1.tar.xz"
  343. if not os.path.exists(f):
  344. self.__download("https://www.python.org/ftp/python/3.4.1/" + f, f)
  345. self.__run_command("tar xf " + f)
  346. self.__run_command("./configure --prefix=$HOME/FrameworkBenchmarks/installs/py3 --disable-shared", cwd="Python-3.4.1")
  347. self.__run_command("make -j4", cwd="Python-3.4.1")
  348. self.__run_command("make install", cwd="Python-3.4.1")
  349. if not os.path.exists("get-pip.py"):
  350. self.__download("https://bootstrap.pypa.io/get-pip.py", "get-pip.py")
  351. self.__run_command("py2/bin/python get-pip.py")
  352. self.__run_command("pypy/bin/pypy get-pip.py")
  353. # Python 3.4.1 installs pip by default.
  354. self.__run_command('py2/bin/pip install -r ../config/requirements.txt')
  355. self.__run_command('py3/bin/pip3 install -r ../config/requirements-py3.txt')
  356. self.__run_command('pypy/bin/pip install -r ../config/requirements-pypy.txt')
  357. ############################################################
  358. # __install_error
  359. ############################################################
  360. def __install_error(self, message):
  361. print("\nINSTALL ERROR: %s\n" % message)
  362. if self.benchmarker.install_error_action == 'abort':
  363. sys.exit("Installation aborted.")
  364. ############################################################
  365. # End __install_error
  366. ############################################################
  367. ############################################################
  368. # __install_database_software
  369. ############################################################
  370. def __install_database_software(self):
  371. print("\nINSTALL: Installing database software\n")
  372. self.__run_command("cd .. && " + self.benchmarker.database_sftp_string(batch_file="../config/database_sftp_batch"), True)
  373. remote_script = """
  374. ##############################
  375. # Prerequisites
  376. ##############################
  377. yes | sudo apt-get update
  378. yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev postgresql
  379. sudo sh -c "echo '* - nofile 65535' >> /etc/security/limits.conf"
  380. sudo mkdir -p /ssd
  381. sudo mkdir -p /ssd/log
  382. ##############################
  383. # MySQL
  384. ##############################
  385. sudo sh -c "echo mysql-server mysql-server/root_password_again select secret | debconf-set-selections"
  386. sudo sh -c "echo mysql-server mysql-server/root_password select secret | debconf-set-selections"
  387. yes | sudo apt-get install mysql-server-5.6
  388. sudo stop mysql
  389. # disable checking of disk size
  390. sudo cp mysql /etc/init.d/mysql
  391. sudo chmod +x /etc/init.d/mysql
  392. sudo cp mysql.conf /etc/init/mysql.conf
  393. # use the my.cnf file to overwrite /etc/mysql/my.cnf
  394. sudo mv /etc/mysql/my.cnf /etc/mysql/my.cnf.orig
  395. sudo mv my.cnf /etc/mysql/my.cnf
  396. sudo cp -R -p /var/lib/mysql /ssd/
  397. sudo cp -R -p /var/log/mysql /ssd/log
  398. sudo cp usr.sbin.mysqld /etc/apparmor.d/
  399. sudo /etc/init.d/apparmor reload
  400. sudo start mysql
  401. # Insert data
  402. mysql -uroot -psecret < create.sql
  403. ##############################
  404. # Postgres
  405. ##############################
  406. sudo useradd benchmarkdbuser -p benchmarkdbpass
  407. sudo -u postgres psql template1 < create-postgres-database.sql
  408. sudo -u benchmarkdbuser psql hello_world < create-postgres.sql
  409. sudo -u postgres -H /etc/init.d/postgresql stop
  410. sudo mv postgresql.conf /etc/postgresql/9.3/main/postgresql.conf
  411. sudo mv pg_hba.conf /etc/postgresql/9.3/main/pg_hba.conf
  412. sudo cp -R -p /var/lib/postgresql/9.3/main /ssd/postgresql
  413. sudo -u postgres -H /etc/init.d/postgresql start
  414. sudo mv 60-postgresql-shm.conf /etc/sysctl.d/60-postgresql-shm.conf
  415. ##############################
  416. # MongoDB
  417. ##############################
  418. sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
  419. sudo cp 10gen.list /etc/apt/sources.list.d/10gen.list
  420. sudo apt-get update
  421. yes | sudo apt-get remove mongodb-clients
  422. yes | sudo apt-get install mongodb-10gen
  423. sudo stop mongodb
  424. sudo mv /etc/mongodb.conf /etc/mongodb.conf.orig
  425. sudo mv mongodb.conf /etc/mongodb.conf
  426. sudo cp -R -p /var/lib/mongodb /ssd/
  427. sudo cp -R -p /var/log/mongodb /ssd/log/
  428. sudo start mongodb
  429. """
  430. print("\nINSTALL: %s" % self.benchmarker.database_ssh_string)
  431. p = subprocess.Popen(self.benchmarker.database_ssh_string.split(" "), stdin=subprocess.PIPE)
  432. p.communicate(remote_script)
  433. returncode = p.returncode
  434. if returncode != 0:
  435. self.__install_error("status code %s running subprocess '%s'." % (returncode, self.benchmarker.database_ssh_string))
  436. print("\nINSTALL: Finished installing database software\n")
  437. ############################################################
  438. # End __install_database_software
  439. ############################################################
  440. ############################################################
  441. # __install_client_software
  442. ############################################################
  443. def __install_client_software(self):
  444. print("\nINSTALL: Installing client software\n")
  445. remote_script = """
  446. ##############################
  447. # Prerequisites
  448. ##############################
  449. yes | sudo apt-get update
  450. yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev
  451. sudo sh -c "echo '* - nofile 65535' >> /etc/security/limits.conf"
  452. ##############################
  453. # wrk
  454. ##############################
  455. git clone https://github.com/wg/wrk.git
  456. cd wrk
  457. make
  458. sudo cp wrk /usr/local/bin
  459. cd ~
  460. #############################
  461. # pipeline.lua
  462. #############################
  463. cat << EOF | tee pipeline.lua
  464. init = function(args)
  465. wrk.init(args)
  466. local r = {}
  467. local depth = tonumber(args[1]) or 1
  468. for i=1,depth do
  469. r[i] = wrk.format()
  470. end
  471. req = table.concat(r)
  472. end
  473. request = function()
  474. return req
  475. end
  476. EOF
  477. """
  478. print("\nINSTALL: %s" % self.benchmarker.client_ssh_string)
  479. p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" "), stdin=subprocess.PIPE)
  480. p.communicate(remote_script)
  481. returncode = p.returncode
  482. if returncode != 0:
  483. self.__install_error("status code %s running subprocess '%s'." % (returncode, self.benchmarker.client_ssh_string))
  484. print("\nINSTALL: Finished installing client software\n")
  485. ############################################################
  486. # End __install_client_software
  487. ############################################################
  488. ############################################################
  489. # __run_command
  490. ############################################################
  491. def __run_command(self, command, send_yes=False, cwd=None, retry=False):
  492. try:
  493. cwd = os.path.join(self.install_dir, cwd)
  494. except AttributeError:
  495. cwd = self.install_dir
  496. if retry:
  497. max_attempts = 5
  498. else:
  499. max_attempts = 1
  500. attempt = 1
  501. delay = 0
  502. if send_yes:
  503. command = "yes yes | " + command
  504. print("\nINSTALL: %s (cwd=%s)" % (command, cwd))
  505. while True:
  506. error_message = ""
  507. try:
  508. # Execute command.
  509. subprocess.check_call(command, shell=True, cwd=cwd)
  510. break # Exit loop if successful.
  511. except:
  512. exceptionType, exceptionValue, exceptionTraceBack = sys.exc_info()
  513. error_message = "".join(traceback.format_exception_only(exceptionType, exceptionValue))
  514. print error_message
  515. # Exit if there are no more attempts left.
  516. attempt += 1
  517. if attempt > max_attempts:
  518. break
  519. # Delay before next attempt.
  520. if delay == 0:
  521. delay = 5
  522. else:
  523. delay = delay * 2
  524. print("Attempt %s/%s starting in %s seconds." % (attempt, max_attempts, delay))
  525. time.sleep(delay)
  526. if error_message:
  527. self.__install_error(error_message)
  528. ############################################################
  529. # End __run_command
  530. ############################################################
  531. ############################################################
  532. # __bash_from_string
  533. # Runs bash -c "command" in install_dir.
  534. ############################################################
  535. def __bash_from_string(self, command):
  536. self.__run_command('bash -c "%s"' % command)
  537. ############################################################
  538. # End __bash_from_string
  539. ############################################################
  540. ############################################################
  541. # __download
  542. # Downloads a file from a URI.
  543. ############################################################
  544. def __download(self, uri, filename=""):
  545. if filename:
  546. if os.path.exists(filename):
  547. return
  548. filename_option = "-O %s " % filename
  549. else:
  550. filename_option = ""
  551. command = "wget -nv --no-check-certificate --trust-server-names %s%s" % (filename_option, uri)
  552. self.__run_command(command, retry=True)
  553. ############################################################
  554. # End __download
  555. ############################################################
  556. ############################################################
  557. # __init__(benchmarker)
  558. ############################################################
  559. def __init__(self, benchmarker):
  560. self.benchmarker = benchmarker
  561. self.install_dir = "installs"
  562. try:
  563. os.mkdir(self.install_dir)
  564. except OSError:
  565. pass
  566. ############################################################
  567. # End __init__
  568. ############################################################
  569. # vim: sw=2