framework_test.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814
  1. from benchmark.fortune_html_parser import FortuneHTMLParser
  2. from setup.linux import setup_util
  3. from benchmark.test_types import *
  4. import importlib
  5. import os
  6. import subprocess
  7. import time
  8. import re
  9. from pprint import pprint
  10. import sys
  11. import traceback
  12. import json
  13. import logging
  14. import csv
  15. import shlex
  16. import math
  17. from collections import OrderedDict
  18. from threading import Thread
  19. from threading import Event
  20. from utils import header
  21. # Cross-platform colored text
  22. from colorama import Fore, Back, Style
  23. class FrameworkTest:
  24. headers_template = "-H 'Host: localhost' -H '{accept}' -H 'Connection: keep-alive'"
  25. # Used for test types that require no pipelining or query string params.
  26. concurrency_template = """
  27. echo ""
  28. echo "---------------------------------------------------------"
  29. echo " Running Primer {name}"
  30. echo " {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
  31. echo "---------------------------------------------------------"
  32. echo ""
  33. {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
  34. sleep 5
  35. echo ""
  36. echo "---------------------------------------------------------"
  37. echo " Running Warmup {name}"
  38. echo " {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}\""
  39. echo "---------------------------------------------------------"
  40. echo ""
  41. {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}"
  42. sleep 5
  43. echo ""
  44. echo "---------------------------------------------------------"
  45. echo " Synchronizing time"
  46. echo "---------------------------------------------------------"
  47. echo ""
  48. ntpdate -s pool.ntp.org
  49. for c in {levels}
  50. do
  51. echo ""
  52. echo "---------------------------------------------------------"
  53. echo " Concurrency: $c for {name}"
  54. echo " {wrk} {headers} -d {duration} -c $c --timeout $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\""
  55. echo "---------------------------------------------------------"
  56. echo ""
  57. STARTTIME=$(date +"%s")
  58. {wrk} {headers} -d {duration} -c $c --timeout $c -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url}
  59. echo "STARTTIME $STARTTIME"
  60. echo "ENDTIME $(date +"%s")"
  61. sleep 2
  62. done
  63. """
  64. # Used for test types that require pipelining.
  65. pipeline_template = """
  66. echo ""
  67. echo "---------------------------------------------------------"
  68. echo " Running Primer {name}"
  69. echo " {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
  70. echo "---------------------------------------------------------"
  71. echo ""
  72. {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
  73. sleep 5
  74. echo ""
  75. echo "---------------------------------------------------------"
  76. echo " Running Warmup {name}"
  77. echo " {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}\""
  78. echo "---------------------------------------------------------"
  79. echo ""
  80. {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}"
  81. sleep 5
  82. echo ""
  83. echo "---------------------------------------------------------"
  84. echo " Synchronizing time"
  85. echo "---------------------------------------------------------"
  86. echo ""
  87. ntpdate -s pool.ntp.org
  88. for c in {levels}
  89. do
  90. echo ""
  91. echo "---------------------------------------------------------"
  92. echo " Concurrency: $c for {name}"
  93. echo " {wrk} {headers} -d {duration} -c $c --timeout $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
  94. echo "---------------------------------------------------------"
  95. echo ""
  96. STARTTIME=$(date +"%s")
  97. {wrk} {headers} -d {duration} -c $c --timeout $c -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
  98. echo "STARTTIME $STARTTIME"
  99. echo "ENDTIME $(date +"%s")"
  100. sleep 2
  101. done
  102. """
  103. # Used for test types that require a database -
  104. # These tests run at a static concurrency level and vary the size of
  105. # the query sent with each request
  106. query_template = """
  107. echo ""
  108. echo "---------------------------------------------------------"
  109. echo " Running Primer {name}"
  110. echo " wrk {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}2\""
  111. echo "---------------------------------------------------------"
  112. echo ""
  113. wrk {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}2"
  114. sleep 5
  115. echo ""
  116. echo "---------------------------------------------------------"
  117. echo " Running Warmup {name}"
  118. echo " wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}2\""
  119. echo "---------------------------------------------------------"
  120. echo ""
  121. wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}2"
  122. sleep 5
  123. echo ""
  124. echo "---------------------------------------------------------"
  125. echo " Synchronizing time"
  126. echo "---------------------------------------------------------"
  127. echo ""
  128. ntpdate -s pool.ntp.org
  129. for c in {levels}
  130. do
  131. echo ""
  132. echo "---------------------------------------------------------"
  133. echo " Queries: $c for {name}"
  134. echo " wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}$c\""
  135. echo "---------------------------------------------------------"
  136. echo ""
  137. STARTTIME=$(date +"%s")
  138. wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}$c"
  139. echo "STARTTIME $STARTTIME"
  140. echo "ENDTIME $(date +"%s")"
  141. sleep 2
  142. done
  143. """
  144. ############################################################
  145. # start(benchmarker)
  146. # Start the test using it's setup file
  147. ############################################################
  148. def start(self, out, err):
  149. # Load profile for this installation
  150. profile="$FWROOT/config/benchmark_profile"
  151. # Setup variables for TROOT and IROOT
  152. setup_util.replace_environ(config=profile,
  153. command='export TROOT=%s && export IROOT=%s && export DBHOST=%s && export MAX_THREADS=%s && export OUT=%s && export ERR=%s' %
  154. (self.directory, self.install_root, self.database_host, self.benchmarker.threads, os.path.join(self.fwroot, out.name), os.path.join(self.fwroot, err.name)))
  155. # Because start can take so long, we print a dot to let the user know
  156. # we are working
  157. class ProgressPrinterThread(Thread):
  158. def __init__(self, event):
  159. Thread.__init__(self)
  160. self.stopped = event
  161. def run(self):
  162. while not self.stopped.wait(20):
  163. sys.stderr.write("Waiting for start to return...\n")
  164. stopFlag = Event()
  165. thread = ProgressPrinterThread(stopFlag)
  166. thread.start()
  167. # Run the module start (inside parent of TROOT)
  168. # - we use the parent as a historical accident - a lot of tests
  169. # use subprocess's cwd argument already
  170. previousDir = os.getcwd()
  171. os.chdir(os.path.dirname(self.troot))
  172. logging.info("Running setup module start (cwd=%s)", self.directory)
  173. # Write the stderr to our temp.txt file to be read and fed back
  174. # to the user via logging later.
  175. with open('temp', 'w') as errout:
  176. # Run the start script for the test as the "testrunner" user.
  177. # This requires superuser privs, so `sudo` is necessary.
  178. # -u [username] The username
  179. # -E Preserves the current environment variables
  180. # -H Forces the home var (~) to be reset to the user specified
  181. # -e Force bash to exit on first error
  182. # Note: check_call is a blocking call, so any startup scripts
  183. # run by the framework that need to continue (read: server has
  184. # started and needs to remain that way), then they should be
  185. # executed in the background.
  186. command = 'sudo -u %s -E -H bash -e %s.sh' % (self.benchmarker.runner_user, self.setup_file)
  187. debug_command = '''\
  188. export FWROOT=%s && \\
  189. export TROOT=%s && \\
  190. export IROOT=%s && \\
  191. export DBHOST=%s && \\
  192. export MAX_THREADS=%s && \\
  193. export OUT=%s && \\
  194. export ERR=%s && \\
  195. cd %s && \\
  196. %s''' % (self.fwroot,
  197. self.directory,
  198. self.install_root,
  199. self.database_host,
  200. self.benchmarker.threads,
  201. os.path.join(self.fwroot, out.name),
  202. os.path.join(self.fwroot, err.name),
  203. self.directory,
  204. command)
  205. logging.info("To run framework manually, copy/paste this:\n%s", debug_command)
  206. try:
  207. subprocess.check_call(command, cwd=self.directory,
  208. shell=True, stderr=errout, stdout=out)
  209. retcode = 0
  210. except Exception:
  211. logging.exception("Failure running setup.sh")
  212. retcode = 1
  213. with open('temp', 'r') as errout:
  214. # Read out temp error output in its entirety
  215. body = errout.read()
  216. if len(body) > 0:
  217. # Log it to the user.
  218. logging.error(body)
  219. # Log it to our err.txt file
  220. err.write(body)
  221. # We are done with our temp file - delete it
  222. os.remove('temp')
  223. os.chdir(previousDir)
  224. # Stop the progress printer
  225. stopFlag.set()
  226. logging.info("Executed %s.sh", self.setup_file)
  227. return retcode
  228. ############################################################
  229. # End start
  230. ############################################################
  231. ############################################################
  232. # verify_urls
  233. # Verifys each of the URLs for this test. THis will sinply
  234. # curl the URL and check for it's return status.
  235. # For each url, a flag will be set on this object for whether
  236. # or not it passed
  237. # Returns True if all verifications succeeded
  238. ############################################################
  239. def verify_urls(self, out, err):
  240. result = True
  241. def verify_type(test_type):
  242. test = self.runTests[test_type]
  243. test.setup_out_err(out, err)
  244. out.write(header("VERIFYING %s" % test_type.upper()))
  245. base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port)
  246. try:
  247. results = test.verify(base_url)
  248. except Exception as e:
  249. results = [('fail',"""Caused Exception in TFB
  250. This almost certainly means your return value is incorrect,
  251. but also that you have found a bug. Please submit an issue
  252. including this message: %s\n%s""" % (e, traceback.format_exc()),
  253. base_url)]
  254. logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
  255. traceback.format_exc()
  256. test.failed = any(result is 'fail' for (result, reason, url) in results)
  257. test.warned = any(result is 'warn' for (result, reason, url) in results)
  258. test.passed = all(result is 'pass' for (result, reason, url) in results)
  259. def output_result(result, reason, url):
  260. prefix = Fore.CYAN
  261. color = Fore.GREEN
  262. if result.upper() == "WARN":
  263. color = Fore.YELLOW
  264. elif result.upper() == "FAIL":
  265. color = Fore.RED
  266. out.write(prefix + " " + color + "%s" + Style.RESET_ALL + " for %s\n" % (result.upper(), url))
  267. print prefix + " " + color + "%s" + Style.RESET_ALL + " for %s" % (result.upper(), url)
  268. if reason is not None and len(reason) != 0:
  269. for line in reason.splitlines():
  270. out.write(" " + line + '\n')
  271. print " " + line
  272. [output_result(r1,r2,url) for (r1, r2, url) in results]
  273. if test.failed:
  274. self.benchmarker.report_verify_results(self, test_type, 'fail')
  275. elif test.warned:
  276. self.benchmarker.report_verify_results(self, test_type, 'warn')
  277. elif test.passed:
  278. self.benchmarker.report_verify_results(self, test_type, 'pass')
  279. else:
  280. raise Exception("Unknown error - test did not pass,warn,or fail")
  281. result = True
  282. for test_type in self.runTests:
  283. verify_type(test_type)
  284. if self.runTests[test_type].failed:
  285. result = False
  286. return result
  287. ############################################################
  288. # End verify_urls
  289. ############################################################
  290. ############################################################
  291. # benchmark
  292. # Runs the benchmark for each type of test that it implements
  293. # JSON/DB/Query.
  294. ############################################################
  295. def benchmark(self, out, err):
  296. def benchmark_type(test_type):
  297. out.write("BENCHMARKING %s ... " % test_type.upper())
  298. test = self.runTests[test_type]
  299. test.setup_out_err(out, err)
  300. output_file = self.benchmarker.output_file(self.name, test_type)
  301. if not os.path.exists(output_file):
  302. # Open to create the empty file
  303. with open(output_file, 'w'):
  304. pass
  305. if not test.failed:
  306. if test_type == 'plaintext': # One special case
  307. remote_script = self.__generate_pipeline_script(test.get_url(), self.port, test.accept_header)
  308. elif test_type == 'query' or test_type == 'update':
  309. remote_script = self.__generate_query_script(test.get_url(), self.port, test.accept_header)
  310. else:
  311. remote_script = self.__generate_concurrency_script(test.get_url(), self.port, test.accept_header)
  312. # Begin resource usage metrics collection
  313. self.__begin_logging(test_type)
  314. # Run the benchmark
  315. with open(output_file, 'w') as raw_file:
  316. p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" "), stdin=subprocess.PIPE, stdout=raw_file, stderr=err)
  317. p.communicate(remote_script)
  318. err.flush()
  319. # End resource usage metrics collection
  320. self.__end_logging()
  321. results = self.__parse_test(test_type)
  322. print "Benchmark results:"
  323. pprint(results)
  324. self.benchmarker.report_benchmark_results(framework=self, test=test_type, results=results['results'])
  325. out.write( "Complete\n" )
  326. out.flush()
  327. for test_type in self.runTests:
  328. benchmark_type(test_type)
  329. ############################################################
  330. # End benchmark
  331. ############################################################
  332. ############################################################
  333. # parse_all
  334. # Method meant to be run for a given timestamp
  335. ############################################################
  336. def parse_all(self):
  337. for test_type in self.runTests:
  338. if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
  339. results = self.__parse_test(test_type)
  340. self.benchmarker.report_benchmark_results(framework=self, test=test_type, results=results['results'])
  341. ############################################################
  342. # __parse_test(test_type)
  343. ############################################################
  344. def __parse_test(self, test_type):
  345. try:
  346. results = dict()
  347. results['results'] = []
  348. stats = []
  349. if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
  350. with open(self.benchmarker.output_file(self.name, test_type)) as raw_data:
  351. is_warmup = True
  352. rawData = None
  353. for line in raw_data:
  354. if "Queries:" in line or "Concurrency:" in line:
  355. is_warmup = False
  356. rawData = None
  357. continue
  358. if "Warmup" in line or "Primer" in line:
  359. is_warmup = True
  360. continue
  361. if not is_warmup:
  362. if rawData == None:
  363. rawData = dict()
  364. results['results'].append(rawData)
  365. #if "Requests/sec:" in line:
  366. # m = re.search("Requests/sec:\s+([0-9]+)", line)
  367. # rawData['reportedResults'] = m.group(1)
  368. # search for weighttp data such as succeeded and failed.
  369. if "Latency" in line:
  370. m = re.findall("([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
  371. if len(m) == 4:
  372. rawData['latencyAvg'] = m[0]
  373. rawData['latencyStdev'] = m[1]
  374. rawData['latencyMax'] = m[2]
  375. # rawData['latencyStdevPercent'] = m[3]
  376. #if "Req/Sec" in line:
  377. # m = re.findall("([0-9]+\.*[0-9]*[k|%]*)", line)
  378. # if len(m) == 4:
  379. # rawData['requestsAvg'] = m[0]
  380. # rawData['requestsStdev'] = m[1]
  381. # rawData['requestsMax'] = m[2]
  382. # rawData['requestsStdevPercent'] = m[3]
  383. #if "requests in" in line:
  384. # m = re.search("requests in ([0-9]+\.*[0-9]*[ms|s|m|h]+)", line)
  385. # if m != None:
  386. # # parse out the raw time, which may be in minutes or seconds
  387. # raw_time = m.group(1)
  388. # if "ms" in raw_time:
  389. # rawData['total_time'] = float(raw_time[:len(raw_time)-2]) / 1000.0
  390. # elif "s" in raw_time:
  391. # rawData['total_time'] = float(raw_time[:len(raw_time)-1])
  392. # elif "m" in raw_time:
  393. # rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 60.0
  394. # elif "h" in raw_time:
  395. # rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 3600.0
  396. if "requests in" in line:
  397. m = re.search("([0-9]+) requests in", line)
  398. if m != None:
  399. rawData['totalRequests'] = int(m.group(1))
  400. if "Socket errors" in line:
  401. if "connect" in line:
  402. m = re.search("connect ([0-9]+)", line)
  403. rawData['connect'] = int(m.group(1))
  404. if "read" in line:
  405. m = re.search("read ([0-9]+)", line)
  406. rawData['read'] = int(m.group(1))
  407. if "write" in line:
  408. m = re.search("write ([0-9]+)", line)
  409. rawData['write'] = int(m.group(1))
  410. if "timeout" in line:
  411. m = re.search("timeout ([0-9]+)", line)
  412. rawData['timeout'] = int(m.group(1))
  413. if "Non-2xx" in line:
  414. m = re.search("Non-2xx or 3xx responses: ([0-9]+)", line)
  415. if m != None:
  416. rawData['5xx'] = int(m.group(1))
  417. if "STARTTIME" in line:
  418. m = re.search("[0-9]+", line)
  419. rawData["startTime"] = int(m.group(0))
  420. if "ENDTIME" in line:
  421. m = re.search("[0-9]+", line)
  422. rawData["endTime"] = int(m.group(0))
  423. test_stats = self.__parse_stats(test_type, rawData["startTime"], rawData["endTime"], 1)
  424. # rawData["averageStats"] = self.__calculate_average_stats(test_stats)
  425. stats.append(test_stats)
  426. with open(self.benchmarker.stats_file(self.name, test_type) + ".json", "w") as stats_file:
  427. json.dump(stats, stats_file, indent=2)
  428. return results
  429. except IOError:
  430. return None
  431. ############################################################
  432. # End benchmark
  433. ############################################################
  434. ##########################################################################################
  435. # Private Methods
  436. ##########################################################################################
  437. ############################################################
  438. # __generate_concurrency_script(url, port)
  439. # Generates the string containing the bash script that will
  440. # be run on the client to benchmark a single test. This
  441. # specifically works for the variable concurrency tests (JSON
  442. # and DB)
  443. ############################################################
  444. def __generate_concurrency_script(self, url, port, accept_header, wrk_command="wrk"):
  445. headers = self.headers_template.format(accept=accept_header)
  446. return self.concurrency_template.format(max_concurrency=max(self.benchmarker.concurrency_levels),
  447. max_threads=self.benchmarker.threads, name=self.name, duration=self.benchmarker.duration,
  448. levels=" ".join("{}".format(item) for item in self.benchmarker.concurrency_levels),
  449. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command)
  450. ############################################################
  451. # __generate_pipeline_script(url, port)
  452. # Generates the string containing the bash script that will
  453. # be run on the client to benchmark a single pipeline test.
  454. ############################################################
  455. def __generate_pipeline_script(self, url, port, accept_header, wrk_command="wrk"):
  456. headers = self.headers_template.format(accept=accept_header)
  457. return self.pipeline_template.format(max_concurrency=16384,
  458. max_threads=self.benchmarker.threads, name=self.name, duration=self.benchmarker.duration,
  459. levels=" ".join("{}".format(item) for item in [256,1024,4096,16384]),
  460. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command,
  461. pipeline=16)
  462. ############################################################
  463. # __generate_query_script(url, port)
  464. # Generates the string containing the bash script that will
  465. # be run on the client to benchmark a single test. This
  466. # specifically works for the variable query tests (Query)
  467. ############################################################
  468. def __generate_query_script(self, url, port, accept_header):
  469. headers = self.headers_template.format(accept=accept_header)
  470. return self.query_template.format(max_concurrency=max(self.benchmarker.concurrency_levels),
  471. max_threads=self.benchmarker.threads, name=self.name, duration=self.benchmarker.duration,
  472. levels=" ".join("{}".format(item) for item in self.benchmarker.query_levels),
  473. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers)
  474. ############################################################
  475. # Returns True if any test type this this framework test will use a DB
  476. ############################################################
  477. def requires_database(self):
  478. '''Returns True/False if this test requires a database'''
  479. return any(tobj.requires_db for (ttype,tobj) in self.runTests.iteritems())
  480. ############################################################
  481. # __begin_logging
  482. # Starts a thread to monitor the resource usage, to be synced with the client's time
  483. # TODO: MySQL and InnoDB are possible. Figure out how to implement them.
  484. ############################################################
  485. def __begin_logging(self, test_type):
  486. output_file = "{file_name}".format(file_name=self.benchmarker.get_stats_file(self.name, test_type))
  487. dstat_string = "dstat -afilmprsT --aio --fs --ipc --lock --raw --socket --tcp \
  488. --raw --socket --tcp --udp --unix --vm --disk-util \
  489. --rpc --rpcd --output {output_file}".format(output_file=output_file)
  490. cmd = shlex.split(dstat_string)
  491. dev_null = open(os.devnull, "w")
  492. self.subprocess_handle = subprocess.Popen(cmd, stdout=dev_null)
  493. ##############################################################
  494. # Begin __end_logging
  495. # Stops the logger thread and blocks until shutdown is complete.
  496. ##############################################################
  497. def __end_logging(self):
  498. self.subprocess_handle.terminate()
  499. self.subprocess_handle.communicate()
  500. ##############################################################
  501. # Begin __parse_stats
  502. # For each test type, process all the statistics, and return a multi-layered dictionary
  503. # that has a structure as follows:
  504. # (timestamp)
  505. # | (main header) - group that the stat is in
  506. # | | (sub header) - title of the stat
  507. # | | | (stat) - the stat itself, usually a floating point number
  508. ##############################################################
  509. def __parse_stats(self, test_type, start_time, end_time, interval):
  510. stats_dict = dict()
  511. stats_file = self.benchmarker.stats_file(self.name, test_type)
  512. with open(stats_file) as stats:
  513. while(stats.next() != "\n"): # dstat doesn't output a completely compliant CSV file - we need to strip the header
  514. pass
  515. stats_reader = csv.reader(stats)
  516. main_header = stats_reader.next()
  517. sub_header = stats_reader.next()
  518. time_row = sub_header.index("epoch")
  519. int_counter = 0
  520. for row in stats_reader:
  521. time = float(row[time_row])
  522. int_counter+=1
  523. if time < start_time:
  524. continue
  525. elif time > end_time:
  526. return stats_dict
  527. if int_counter % interval != 0:
  528. continue
  529. row_dict = dict()
  530. for nextheader in main_header:
  531. if nextheader != "":
  532. row_dict[nextheader] = dict()
  533. header = ""
  534. for item_num, column in enumerate(row):
  535. if(len(main_header[item_num]) != 0):
  536. header = main_header[item_num]
  537. row_dict[header][sub_header[item_num]] = float(column) # all the stats are numbers, so we want to make sure that they stay that way in json
  538. stats_dict[time] = row_dict
  539. return stats_dict
  540. ##############################################################
  541. # End __parse_stats
  542. ##############################################################
  543. def __getattr__(self, name):
  544. """For backwards compatibility, we used to pass benchmarker
  545. as the argument to the setup.sh files"""
  546. try:
  547. x = getattr(self.benchmarker, name)
  548. except AttributeError:
  549. print "AttributeError: %s not a member of FrameworkTest or Benchmarker" % name
  550. print "This is probably a bug"
  551. raise
  552. return x
  553. ##############################################################
  554. # Begin __calculate_average_stats
  555. # We have a large amount of raw data for the statistics that
  556. # may be useful for the stats nerds, but most people care about
  557. # a couple of numbers. For now, we're only going to supply:
  558. # * Average CPU
  559. # * Average Memory
  560. # * Total network use
  561. # * Total disk use
  562. # More may be added in the future. If they are, please update
  563. # the above list.
  564. # Note: raw_stats is directly from the __parse_stats method.
  565. # Recall that this consists of a dictionary of timestamps,
  566. # each of which contain a dictionary of stat categories which
  567. # contain a dictionary of stats
  568. ##############################################################
  569. def __calculate_average_stats(self, raw_stats):
  570. raw_stat_collection = dict()
  571. for timestamp, time_dict in raw_stats.items():
  572. for main_header, sub_headers in time_dict.items():
  573. item_to_append = None
  574. if 'cpu' in main_header:
  575. # We want to take the idl stat and subtract it from 100
  576. # to get the time that the CPU is NOT idle.
  577. item_to_append = sub_headers['idl'] - 100.0
  578. elif main_header == 'memory usage':
  579. item_to_append = sub_headers['used']
  580. elif 'net' in main_header:
  581. # Network stats have two parts - recieve and send. We'll use a tuple of
  582. # style (recieve, send)
  583. item_to_append = (sub_headers['recv'], sub_headers['send'])
  584. elif 'dsk' or 'io' in main_header:
  585. # Similar for network, except our tuple looks like (read, write)
  586. item_to_append = (sub_headers['read'], sub_headers['writ'])
  587. if item_to_append is not None:
  588. if main_header not in raw_stat_collection:
  589. raw_stat_collection[main_header] = list()
  590. raw_stat_collection[main_header].append(item_to_append)
  591. # Simple function to determine human readable size
  592. # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
  593. def sizeof_fmt(num):
  594. # We'll assume that any number we get is convertable to a float, just in case
  595. num = float(num)
  596. for x in ['bytes','KB','MB','GB']:
  597. if num < 1024.0 and num > -1024.0:
  598. return "%3.1f%s" % (num, x)
  599. num /= 1024.0
  600. return "%3.1f%s" % (num, 'TB')
  601. # Now we have our raw stats in a readable format - we need to format it for display
  602. # We need a floating point sum, so the built in sum doesn't cut it
  603. display_stat_collection = dict()
  604. for header, values in raw_stat_collection.items():
  605. display_stat = None
  606. if 'cpu' in header:
  607. display_stat = sizeof_fmt(math.fsum(values) / len(values))
  608. elif main_header == 'memory usage':
  609. display_stat = sizeof_fmt(math.fsum(values) / len(values))
  610. elif 'net' in main_header:
  611. receive, send = zip(*values) # unzip
  612. display_stat = {'receive': sizeof_fmt(math.fsum(receive)), 'send': sizeof_fmt(math.fsum(send))}
  613. else: # if 'dsk' or 'io' in header:
  614. read, write = zip(*values) # unzip
  615. display_stat = {'read': sizeof_fmt(math.fsum(read)), 'write': sizeof_fmt(math.fsum(write))}
  616. display_stat_collection[header] = display_stat
  617. return display_stat
  618. ###########################################################################################
  619. # End __calculate_average_stats
  620. #########################################################################################
  621. ##########################################################################################
  622. # Constructor
  623. ##########################################################################################
  624. def __init__(self, name, directory, benchmarker, runTests, args):
  625. self.name = name
  626. self.directory = directory
  627. self.benchmarker = benchmarker
  628. self.runTests = runTests
  629. self.fwroot = benchmarker.fwroot
  630. self.approach = ""
  631. self.classification = ""
  632. self.database = ""
  633. self.framework = ""
  634. self.language = ""
  635. self.orm = ""
  636. self.platform = ""
  637. self.webserver = ""
  638. self.os = ""
  639. self.database_os = ""
  640. self.display_name = ""
  641. self.notes = ""
  642. self.versus = ""
  643. # setup logging
  644. logging.basicConfig(stream=sys.stderr, level=logging.INFO)
  645. self.install_root="%s/%s" % (self.fwroot, "installs")
  646. if benchmarker.install_strategy is 'pertest':
  647. self.install_root="%s/pertest/%s" % (self.install_root, name)
  648. # Used in setup.sh scripts for consistency with
  649. # the bash environment variables
  650. self.troot = self.directory
  651. self.iroot = self.install_root
  652. self.__dict__.update(args)
  653. ############################################################
  654. # End __init__
  655. ############################################################
  656. ############################################################
  657. # End FrameworkTest
  658. ############################################################
  659. ##########################################################################################
  660. # Static methods
  661. ##########################################################################################
  662. ##############################################################
  663. # parse_config(config, directory, benchmarker)
  664. # parses a config file and returns a list of FrameworkTest
  665. # objects based on that config file.
  666. ##############################################################
  667. def parse_config(config, directory, benchmarker):
  668. tests = []
  669. # This sort ordering is set up specifically to return the length
  670. # of the test name. There were SO many problems involved with
  671. # 'plaintext' being run first (rather, just not last) that we
  672. # needed to ensure that it was run last for every framework.
  673. def testOrder(type_name):
  674. return len(type_name)
  675. # The config object can specify multiple tests
  676. # Loop over them and parse each into a FrameworkTest
  677. for test in config['tests']:
  678. names = [name for (name,keys) in test.iteritems()]
  679. if "default" not in names:
  680. logging.warn("Framework %s does not define a default test in benchmark_config.json", config['framework'])
  681. for test_name, test_keys in test.iteritems():
  682. # Prefix all test names with framework except 'default' test
  683. if test_name == 'default':
  684. test_name = config['framework']
  685. else:
  686. test_name = "%s-%s" % (config['framework'], test_name)
  687. # Ensure FrameworkTest.framework is available
  688. if not test_keys['framework']:
  689. test_keys['framework'] = config['framework']
  690. #if test_keys['framework'].lower() != config['framework'].lower():
  691. # print Exception("benchmark_config.json for test %s is invalid - test framework '%s' must match benchmark_config.json framework '%s'" %
  692. # (test_name, test_keys['framework'], config['framework']))
  693. # Confirm required keys are present
  694. # TODO have a TechEmpower person confirm this list - I don't know what the website requires....
  695. required = ['language','webserver','classification','database','approach','orm','framework','os','database_os']
  696. if not all (key in test_keys for key in required):
  697. raise Exception("benchmark_config.json for test %s is invalid - missing required keys" % test_name)
  698. # Map test type to a parsed FrameworkTestType object
  699. runTests = dict()
  700. for type_name, type_obj in benchmarker.types.iteritems():
  701. try:
  702. runTests[type_name] = type_obj.copy().parse(test_keys)
  703. except AttributeError as ae:
  704. # This is quite common - most tests don't support all types
  705. # Quitely log it and move on (debug logging is on in travis and this causes
  706. # ~1500 lines of debug, so I'm totally ignoring it for now
  707. # logging.debug("Missing arguments for test type %s for framework test %s", type_name, test_name)
  708. pass
  709. # We need to sort by test_type to run
  710. sortedTestKeys = sorted(runTests.keys(), key=testOrder)
  711. sortedRunTests = OrderedDict()
  712. for sortedTestKey in sortedTestKeys:
  713. sortedRunTests[sortedTestKey] = runTests[sortedTestKey]
  714. # By passing the entire set of keys, each FrameworkTest will have a member for each key
  715. tests.append(FrameworkTest(test_name, directory, benchmarker, sortedRunTests, test_keys))
  716. return tests
  717. ##############################################################
  718. # End parse_config
  719. ##############################################################