framework_test.py 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041
  1. from benchmark.fortune_html_parser import FortuneHTMLParser
  2. from setup.linux import setup_util
  3. from benchmark.test_types import *
  4. import importlib
  5. import os
  6. import subprocess
  7. import time
  8. import re
  9. from pprint import pprint
  10. import sys
  11. import traceback
  12. import json
  13. import logging
  14. import csv
  15. import shlex
  16. import math
  17. from collections import OrderedDict
  18. from requests import ConnectionError
  19. from threading import Thread
  20. from threading import Event
  21. from utils import header
  22. # Cross-platform colored text
  23. from colorama import Fore, Back, Style
  24. from datetime import datetime
  25. from datetime import timedelta
  26. class FrameworkTest:
  27. headers_template = "-H 'Host: localhost' -H 'Accept: {accept}' -H 'Connection: keep-alive'"
  28. # Used for test types that require no pipelining or query string params.
  29. concurrency_template = """
  30. echo ""
  31. echo "---------------------------------------------------------"
  32. echo " Running Primer {name}"
  33. echo " {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
  34. echo "---------------------------------------------------------"
  35. echo ""
  36. {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
  37. sleep 5
  38. echo ""
  39. echo "---------------------------------------------------------"
  40. echo " Running Warmup {name}"
  41. echo " {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} \"http://{server_host}:{port}{url}\""
  42. echo "---------------------------------------------------------"
  43. echo ""
  44. {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} "http://{server_host}:{port}{url}"
  45. sleep 5
  46. echo ""
  47. echo "---------------------------------------------------------"
  48. echo " Synchronizing time"
  49. echo "---------------------------------------------------------"
  50. echo ""
  51. ntpdate -s pool.ntp.org
  52. for c in {levels}
  53. do
  54. echo ""
  55. echo "---------------------------------------------------------"
  56. echo " Concurrency: $c for {name}"
  57. echo " {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\""
  58. echo "---------------------------------------------------------"
  59. echo ""
  60. STARTTIME=$(date +"%s")
  61. {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url}
  62. echo "STARTTIME $STARTTIME"
  63. echo "ENDTIME $(date +"%s")"
  64. sleep 2
  65. done
  66. """
  67. # Used for test types that require pipelining.
  68. pipeline_template = """
  69. echo ""
  70. echo "---------------------------------------------------------"
  71. echo " Running Primer {name}"
  72. echo " {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
  73. echo "---------------------------------------------------------"
  74. echo ""
  75. {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
  76. sleep 5
  77. echo ""
  78. echo "---------------------------------------------------------"
  79. echo " Running Warmup {name}"
  80. echo " {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} \"http://{server_host}:{port}{url}\""
  81. echo "---------------------------------------------------------"
  82. echo ""
  83. {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} "http://{server_host}:{port}{url}"
  84. sleep 5
  85. echo ""
  86. echo "---------------------------------------------------------"
  87. echo " Synchronizing time"
  88. echo "---------------------------------------------------------"
  89. echo ""
  90. ntpdate -s pool.ntp.org
  91. for c in {levels}
  92. do
  93. echo ""
  94. echo "---------------------------------------------------------"
  95. echo " Concurrency: $c for {name}"
  96. echo " {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
  97. echo "---------------------------------------------------------"
  98. echo ""
  99. STARTTIME=$(date +"%s")
  100. {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
  101. echo "STARTTIME $STARTTIME"
  102. echo "ENDTIME $(date +"%s")"
  103. sleep 2
  104. done
  105. """
  106. # Used for test types that require a database -
  107. # These tests run at a static concurrency level and vary the size of
  108. # the query sent with each request
  109. query_template = """
  110. echo ""
  111. echo "---------------------------------------------------------"
  112. echo " Running Primer {name}"
  113. echo " wrk {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}2\""
  114. echo "---------------------------------------------------------"
  115. echo ""
  116. wrk {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}2"
  117. sleep 5
  118. echo ""
  119. echo "---------------------------------------------------------"
  120. echo " Running Warmup {name}"
  121. echo " wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} \"http://{server_host}:{port}{url}2\""
  122. echo "---------------------------------------------------------"
  123. echo ""
  124. wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} "http://{server_host}:{port}{url}2"
  125. sleep 5
  126. echo ""
  127. echo "---------------------------------------------------------"
  128. echo " Synchronizing time"
  129. echo "---------------------------------------------------------"
  130. echo ""
  131. ntpdate -s pool.ntp.org
  132. for c in {levels}
  133. do
  134. echo ""
  135. echo "---------------------------------------------------------"
  136. echo " Queries: $c for {name}"
  137. echo " wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} \"http://{server_host}:{port}{url}$c\""
  138. echo "---------------------------------------------------------"
  139. echo ""
  140. STARTTIME=$(date +"%s")
  141. wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t {max_threads} "http://{server_host}:{port}{url}$c"
  142. echo "STARTTIME $STARTTIME"
  143. echo "ENDTIME $(date +"%s")"
  144. sleep 2
  145. done
  146. """
  147. ############################################################
  148. # start(benchmarker)
  149. # Start the test using its setup file
  150. ############################################################
  151. def start(self, out):
  152. # Setup environment variables
  153. logDir = os.path.join(self.fwroot, self.benchmarker.full_results_directory(), 'logs', self.name.lower())
  154. bash_functions_path= os.path.join(self.fwroot, 'toolset/setup/linux/bash_functions.sh')
  155. setup_util.replace_environ(config='$FWROOT/config/benchmark_profile',
  156. command='''\
  157. export TROOT=%s && \
  158. export IROOT=%s && \
  159. export DBHOST=%s && \
  160. export LOGDIR=%s && \
  161. export MAX_THREADS=%s && \
  162. export MAX_CONCURRENCY=%s \
  163. ''' % (
  164. self.directory,
  165. self.install_root,
  166. self.database_host,
  167. logDir,
  168. self.benchmarker.threads,
  169. max(self.benchmarker.concurrency_levels)))
  170. # Always ensure that IROOT exists
  171. if not os.path.exists(self.install_root):
  172. os.mkdir(self.install_root)
  173. # Run the module start inside parent of TROOT
  174. # - we use the parent as a historical accident, a number of tests
  175. # refer to their TROOT maually still
  176. previousDir = os.getcwd()
  177. os.chdir(os.path.dirname(self.troot))
  178. logging.info("Running setup module start (cwd=%s)", self.directory)
  179. command = 'bash -exc "source %s && source %s.sh"' % (
  180. bash_functions_path,
  181. os.path.join(self.troot, self.setup_file))
  182. debug_command = '''\
  183. export FWROOT=%s && \\
  184. export TROOT=%s && \\
  185. export IROOT=%s && \\
  186. export DBHOST=%s && \\
  187. export LOGDIR=%s && \\
  188. export MAX_THREADS=%s && \\
  189. export MAX_CONCURRENCY=%s && \\
  190. cd %s && \\
  191. %s''' % (self.fwroot,
  192. self.directory,
  193. self.install_root,
  194. self.database_host,
  195. logDir,
  196. self.benchmarker.threads,
  197. max(self.benchmarker.concurrency_levels),
  198. self.directory,
  199. command)
  200. logging.info("To run %s manually, copy/paste this:\n%s", self.name, debug_command)
  201. def tee_output(prefix, line):
  202. # Needs to be one atomic write
  203. # Explicitly use UTF-8 as it's the most common framework output
  204. # TODO improve encoding handling
  205. line = prefix.encode('utf-8') + line
  206. # Log to current terminal
  207. sys.stdout.write(line)
  208. sys.stdout.flush()
  209. # logging.error("".join([prefix, line]))
  210. out.write(line)
  211. out.flush()
  212. # Start the setup.sh command
  213. p = subprocess.Popen(command, cwd=self.directory,
  214. shell=True, stdout=subprocess.PIPE,
  215. stderr=subprocess.STDOUT)
  216. pgid = os.getpgid(p.pid)
  217. nbsr = setup_util.NonBlockingStreamReader(p.stdout,
  218. "%s: %s.sh and framework processes have terminated" % (self.name, self.setup_file))
  219. # Set a limit on total execution time of setup.sh
  220. timeout = datetime.now() + timedelta(minutes = 105)
  221. time_remaining = timeout - datetime.now()
  222. # Need to print to stdout once every 10 minutes or Travis-CI will abort
  223. travis_timeout = datetime.now() + timedelta(minutes = 5)
  224. # Flush output until setup.sh work is finished. This is
  225. # either a) when setup.sh exits b) when the port is bound
  226. # c) when we run out of time. Note that 'finished' doesn't
  227. # guarantee setup.sh process is dead - the OS may choose to make
  228. # setup.sh a zombie process if it still has living children
  229. #
  230. # Note: child processes forked (using &) will remain alive
  231. # after setup.sh has exited. The will have inherited the
  232. # stdout/stderr descriptors and will be directing their
  233. # output to the pipes.
  234. #
  235. prefix = "Setup %s: " % self.name
  236. while (p.poll() is None
  237. and not self.benchmarker.is_port_bound(self.port)
  238. and not time_remaining.total_seconds() < 0):
  239. # The conditions above are slow to check, so
  240. # we will delay output substantially if we only
  241. # print one line per condition check.
  242. # Adding a tight loop here mitigates the effect,
  243. # ensuring that most of the output directly from
  244. # setup.sh is sent to tee_output before the outer
  245. # loop exits and prints things like "setup.sh exited"
  246. #
  247. for i in xrange(10):
  248. try:
  249. line = nbsr.readline(0.05)
  250. if line:
  251. tee_output(prefix, line)
  252. # Reset Travis-CI timer
  253. travis_timeout = datetime.now() + timedelta(minutes = 5)
  254. except setup_util.EndOfStream:
  255. tee_output(prefix, "Setup has terminated\n")
  256. break
  257. time_remaining = timeout - datetime.now()
  258. if (travis_timeout - datetime.now()).total_seconds() < 0:
  259. sys.stdout.write(prefix + 'Printing so Travis-CI does not time out\n')
  260. sys.stdout.write(prefix + "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
  261. p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
  262. sys.stdout.flush()
  263. travis_timeout = datetime.now() + timedelta(minutes = 5)
  264. # Did we time out?
  265. if time_remaining.total_seconds() < 0:
  266. tee_output(prefix, "%s.sh timed out!! Aborting...\n" % self.setup_file)
  267. p.kill()
  268. return 1
  269. # What's our return code?
  270. # If setup.sh has terminated, use that code
  271. # Otherwise, detect if the port was bound
  272. tee_output(prefix, "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
  273. p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
  274. retcode = (p.poll() if p.poll() is not None else 0 if self.benchmarker.is_port_bound(self.port) else 1)
  275. if p.poll() is not None:
  276. tee_output(prefix, "%s.sh process exited naturally with %s\n" % (self.setup_file, p.poll()))
  277. elif self.benchmarker.is_port_bound(self.port):
  278. tee_output(prefix, "Bound port detected on %s\n" % self.port)
  279. # Before we return control to the benchmarker, spin up a
  280. # thread to keep an eye on the pipes in case the running
  281. # framework uses stdout/stderr. Once all processes accessing
  282. # the subprocess.PIPEs are dead, this thread will terminate.
  283. # Use a different prefix to indicate this is the framework
  284. # speaking
  285. prefix = "Server %s: " % self.name
  286. def watch_child_pipes(nbsr, prefix):
  287. while True:
  288. try:
  289. line = nbsr.readline(60)
  290. if line:
  291. tee_output(prefix, line)
  292. except setup_util.EndOfStream:
  293. tee_output(prefix, "Framework processes have terminated\n")
  294. return
  295. watch_thread = Thread(target = watch_child_pipes,
  296. args = (nbsr, prefix))
  297. watch_thread.daemon = True
  298. watch_thread.start()
  299. logging.info("Executed %s.sh, returning %s", self.setup_file, retcode)
  300. os.chdir(previousDir)
  301. return retcode, pgid
  302. ############################################################
  303. # End start
  304. ############################################################
  305. ############################################################
  306. # verify_urls
  307. # Verifys each of the URLs for this test. THis will sinply
  308. # curl the URL and check for it's return status.
  309. # For each url, a flag will be set on this object for whether
  310. # or not it passed
  311. # Returns True if all verifications succeeded
  312. ############################################################
  313. def verify_urls(self, logPath):
  314. result = True
  315. def verify_type(test_type):
  316. verificationPath = os.path.join(logPath, test_type)
  317. try:
  318. os.makedirs(verificationPath)
  319. except OSError:
  320. pass
  321. with open(os.path.join(verificationPath, 'verification.txt'), 'w') as verification:
  322. test = self.runTests[test_type]
  323. test.setup_out(verification)
  324. verification.write(header("VERIFYING %s" % test_type.upper()))
  325. base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port)
  326. try:
  327. results = test.verify(base_url)
  328. except ConnectionError as e:
  329. results = [('fail',"Server did not respond to request", base_url)]
  330. logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
  331. except Exception as e:
  332. results = [('fail',"""Caused Exception in TFB
  333. This almost certainly means your return value is incorrect,
  334. but also that you have found a bug. Please submit an issue
  335. including this message: %s\n%s""" % (e, traceback.format_exc()),
  336. base_url)]
  337. logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
  338. traceback.format_exc()
  339. test.failed = any(result == 'fail' for (result, reason, url) in results)
  340. test.warned = any(result == 'warn' for (result, reason, url) in results)
  341. test.passed = all(result == 'pass' for (result, reason, url) in results)
  342. def output_result(result, reason, url):
  343. specific_rules_url = "http://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/#specific-test-requirements"
  344. color = Fore.GREEN
  345. if result.upper() == "WARN":
  346. color = Fore.YELLOW
  347. elif result.upper() == "FAIL":
  348. color = Fore.RED
  349. verification.write((" " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url))
  350. print (" " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url)
  351. if reason is not None and len(reason) != 0:
  352. for line in reason.splitlines():
  353. verification.write(" " + line + '\n')
  354. print " " + line
  355. if not test.passed:
  356. verification.write(" See %s\n" % specific_rules_url)
  357. print " See %s\n" % specific_rules_url
  358. [output_result(r1,r2,url) for (r1, r2, url) in results]
  359. if test.failed:
  360. self.benchmarker.report_verify_results(self, test_type, 'fail')
  361. elif test.warned:
  362. self.benchmarker.report_verify_results(self, test_type, 'warn')
  363. elif test.passed:
  364. self.benchmarker.report_verify_results(self, test_type, 'pass')
  365. else:
  366. raise Exception("Unknown error - test did not pass,warn,or fail")
  367. verification.flush()
  368. result = True
  369. for test_type in self.runTests:
  370. verify_type(test_type)
  371. if self.runTests[test_type].failed:
  372. result = False
  373. return result
  374. ############################################################
  375. # End verify_urls
  376. ############################################################
  377. ############################################################
  378. # benchmark
  379. # Runs the benchmark for each type of test that it implements
  380. # JSON/DB/Query.
  381. ############################################################
  382. def benchmark(self, logPath):
  383. def benchmark_type(test_type):
  384. benchmarkPath = os.path.join(logPath, test_type)
  385. try:
  386. os.makedirs(benchmarkPath)
  387. except OSError:
  388. pass
  389. with open(os.path.join(benchmarkPath, 'benchmark.txt'), 'w') as out:
  390. out.write("BENCHMARKING %s ... " % test_type.upper())
  391. test = self.runTests[test_type]
  392. test.setup_out(out)
  393. output_file = self.benchmarker.output_file(self.name, test_type)
  394. if not os.path.exists(output_file):
  395. # Open to create the empty file
  396. with open(output_file, 'w'):
  397. pass
  398. if not test.failed:
  399. if test_type == 'plaintext': # One special case
  400. remote_script = self.__generate_pipeline_script(test.get_url(), self.port, test.accept_header)
  401. elif test_type == 'query' or test_type == 'update':
  402. remote_script = self.__generate_query_script(test.get_url(), self.port, test.accept_header)
  403. else:
  404. remote_script = self.__generate_concurrency_script(test.get_url(), self.port, test.accept_header)
  405. # Begin resource usage metrics collection
  406. self.__begin_logging(test_type)
  407. # Run the benchmark
  408. with open(output_file, 'w') as raw_file:
  409. p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" "), stdin=subprocess.PIPE, stdout=raw_file, stderr=raw_file)
  410. p.communicate(remote_script)
  411. out.flush()
  412. # End resource usage metrics collection
  413. self.__end_logging()
  414. results = self.__parse_test(test_type)
  415. print "Benchmark results:"
  416. pprint(results)
  417. self.benchmarker.report_benchmark_results(framework=self, test=test_type, results=results['results'])
  418. out.write( "Complete\n" )
  419. out.flush()
  420. for test_type in self.runTests:
  421. benchmark_type(test_type)
  422. ############################################################
  423. # End benchmark
  424. ############################################################
  425. ############################################################
  426. # parse_all
  427. # Method meant to be run for a given timestamp
  428. ############################################################
  429. def parse_all(self):
  430. for test_type in self.runTests:
  431. if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
  432. results = self.__parse_test(test_type)
  433. self.benchmarker.report_benchmark_results(framework=self, test=test_type, results=results['results'])
  434. ##########################################################################################
  435. # Private Methods
  436. ##########################################################################################
  437. ############################################################
  438. # __parse_test(test_type)
  439. ############################################################
  440. def __parse_test(self, test_type):
  441. try:
  442. results = dict()
  443. results['results'] = []
  444. stats = []
  445. if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
  446. with open(self.benchmarker.output_file(self.name, test_type)) as raw_data:
  447. is_warmup = True
  448. rawData = None
  449. for line in raw_data:
  450. if "Queries:" in line or "Concurrency:" in line:
  451. is_warmup = False
  452. rawData = None
  453. continue
  454. if "Warmup" in line or "Primer" in line:
  455. is_warmup = True
  456. continue
  457. if not is_warmup:
  458. if rawData == None:
  459. rawData = dict()
  460. results['results'].append(rawData)
  461. #if "Requests/sec:" in line:
  462. # m = re.search("Requests/sec:\s+([0-9]+)", line)
  463. # rawData['reportedResults'] = m.group(1)
  464. # search for weighttp data such as succeeded and failed.
  465. if "Latency" in line:
  466. m = re.findall("([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
  467. if len(m) == 4:
  468. rawData['latencyAvg'] = m[0]
  469. rawData['latencyStdev'] = m[1]
  470. rawData['latencyMax'] = m[2]
  471. # rawData['latencyStdevPercent'] = m[3]
  472. #if "Req/Sec" in line:
  473. # m = re.findall("([0-9]+\.*[0-9]*[k|%]*)", line)
  474. # if len(m) == 4:
  475. # rawData['requestsAvg'] = m[0]
  476. # rawData['requestsStdev'] = m[1]
  477. # rawData['requestsMax'] = m[2]
  478. # rawData['requestsStdevPercent'] = m[3]
  479. #if "requests in" in line:
  480. # m = re.search("requests in ([0-9]+\.*[0-9]*[ms|s|m|h]+)", line)
  481. # if m != None:
  482. # # parse out the raw time, which may be in minutes or seconds
  483. # raw_time = m.group(1)
  484. # if "ms" in raw_time:
  485. # rawData['total_time'] = float(raw_time[:len(raw_time)-2]) / 1000.0
  486. # elif "s" in raw_time:
  487. # rawData['total_time'] = float(raw_time[:len(raw_time)-1])
  488. # elif "m" in raw_time:
  489. # rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 60.0
  490. # elif "h" in raw_time:
  491. # rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 3600.0
  492. if "requests in" in line:
  493. m = re.search("([0-9]+) requests in", line)
  494. if m != None:
  495. rawData['totalRequests'] = int(m.group(1))
  496. if "Socket errors" in line:
  497. if "connect" in line:
  498. m = re.search("connect ([0-9]+)", line)
  499. rawData['connect'] = int(m.group(1))
  500. if "read" in line:
  501. m = re.search("read ([0-9]+)", line)
  502. rawData['read'] = int(m.group(1))
  503. if "write" in line:
  504. m = re.search("write ([0-9]+)", line)
  505. rawData['write'] = int(m.group(1))
  506. if "timeout" in line:
  507. m = re.search("timeout ([0-9]+)", line)
  508. rawData['timeout'] = int(m.group(1))
  509. if "Non-2xx" in line:
  510. m = re.search("Non-2xx or 3xx responses: ([0-9]+)", line)
  511. if m != None:
  512. rawData['5xx'] = int(m.group(1))
  513. if "STARTTIME" in line:
  514. m = re.search("[0-9]+", line)
  515. rawData["startTime"] = int(m.group(0))
  516. if "ENDTIME" in line:
  517. m = re.search("[0-9]+", line)
  518. rawData["endTime"] = int(m.group(0))
  519. test_stats = self.__parse_stats(test_type, rawData["startTime"], rawData["endTime"], 1)
  520. # rawData["averageStats"] = self.__calculate_average_stats(test_stats)
  521. stats.append(test_stats)
  522. with open(self.benchmarker.stats_file(self.name, test_type) + ".json", "w") as stats_file:
  523. json.dump(stats, stats_file, indent=2)
  524. return results
  525. except IOError:
  526. return None
  527. ############################################################
  528. # End benchmark
  529. ############################################################
  530. ############################################################
  531. # __generate_concurrency_script(url, port)
  532. # Generates the string containing the bash script that will
  533. # be run on the client to benchmark a single test. This
  534. # specifically works for the variable concurrency tests (JSON
  535. # and DB)
  536. ############################################################
  537. def __generate_concurrency_script(self, url, port, accept_header, wrk_command="wrk"):
  538. headers = self.headers_template.format(accept=accept_header)
  539. return self.concurrency_template.format(max_concurrency=max(self.benchmarker.concurrency_levels),
  540. max_threads=self.benchmarker.threads, name=self.name, duration=self.benchmarker.duration,
  541. levels=" ".join("{}".format(item) for item in self.benchmarker.concurrency_levels),
  542. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command)
  543. ############################################################
  544. # __generate_pipeline_script(url, port)
  545. # Generates the string containing the bash script that will
  546. # be run on the client to benchmark a single pipeline test.
  547. ############################################################
  548. def __generate_pipeline_script(self, url, port, accept_header, wrk_command="wrk"):
  549. headers = self.headers_template.format(accept=accept_header)
  550. return self.pipeline_template.format(max_concurrency=16384,
  551. max_threads=self.benchmarker.threads, name=self.name, duration=self.benchmarker.duration,
  552. levels=" ".join("{}".format(item) for item in [256,1024,4096,16384]),
  553. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command,
  554. pipeline=16)
  555. ############################################################
  556. # __generate_query_script(url, port)
  557. # Generates the string containing the bash script that will
  558. # be run on the client to benchmark a single test. This
  559. # specifically works for the variable query tests (Query)
  560. ############################################################
  561. def __generate_query_script(self, url, port, accept_header):
  562. headers = self.headers_template.format(accept=accept_header)
  563. return self.query_template.format(max_concurrency=max(self.benchmarker.concurrency_levels),
  564. max_threads=self.benchmarker.threads, name=self.name, duration=self.benchmarker.duration,
  565. levels=" ".join("{}".format(item) for item in self.benchmarker.query_levels),
  566. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers)
  567. ############################################################
  568. # Returns True if any test type this this framework test will use a DB
  569. ############################################################
  570. def requires_database(self):
  571. '''Returns True/False if this test requires a database'''
  572. return any(tobj.requires_db for (ttype,tobj) in self.runTests.iteritems())
  573. ############################################################
  574. # __begin_logging
  575. # Starts a thread to monitor the resource usage, to be synced with the client's time
  576. # TODO: MySQL and InnoDB are possible. Figure out how to implement them.
  577. ############################################################
  578. def __begin_logging(self, test_type):
  579. output_file = "{file_name}".format(file_name=self.benchmarker.get_stats_file(self.name, test_type))
  580. dstat_string = "dstat -afilmprsT --aio --fs --ipc --lock --raw --socket --tcp \
  581. --raw --socket --tcp --udp --unix --vm --disk-util \
  582. --rpc --rpcd --output {output_file}".format(output_file=output_file)
  583. cmd = shlex.split(dstat_string)
  584. dev_null = open(os.devnull, "w")
  585. self.subprocess_handle = subprocess.Popen(cmd, stdout=dev_null)
  586. ##############################################################
  587. # Begin __end_logging
  588. # Stops the logger thread and blocks until shutdown is complete.
  589. ##############################################################
  590. def __end_logging(self):
  591. self.subprocess_handle.terminate()
  592. self.subprocess_handle.communicate()
  593. ##############################################################
  594. # Begin __parse_stats
  595. # For each test type, process all the statistics, and return a multi-layered dictionary
  596. # that has a structure as follows:
  597. # (timestamp)
  598. # | (main header) - group that the stat is in
  599. # | | (sub header) - title of the stat
  600. # | | | (stat) - the stat itself, usually a floating point number
  601. ##############################################################
  602. def __parse_stats(self, test_type, start_time, end_time, interval):
  603. stats_dict = dict()
  604. stats_file = self.benchmarker.stats_file(self.name, test_type)
  605. with open(stats_file) as stats:
  606. while(stats.next() != "\n"): # dstat doesn't output a completely compliant CSV file - we need to strip the header
  607. pass
  608. stats_reader = csv.reader(stats)
  609. main_header = stats_reader.next()
  610. sub_header = stats_reader.next()
  611. time_row = sub_header.index("epoch")
  612. int_counter = 0
  613. for row in stats_reader:
  614. time = float(row[time_row])
  615. int_counter+=1
  616. if time < start_time:
  617. continue
  618. elif time > end_time:
  619. return stats_dict
  620. if int_counter % interval != 0:
  621. continue
  622. row_dict = dict()
  623. for nextheader in main_header:
  624. if nextheader != "":
  625. row_dict[nextheader] = dict()
  626. header = ""
  627. for item_num, column in enumerate(row):
  628. if(len(main_header[item_num]) != 0):
  629. header = main_header[item_num]
  630. row_dict[header][sub_header[item_num]] = float(column) # all the stats are numbers, so we want to make sure that they stay that way in json
  631. stats_dict[time] = row_dict
  632. return stats_dict
  633. ##############################################################
  634. # End __parse_stats
  635. ##############################################################
  636. def __getattr__(self, name):
  637. """For backwards compatibility, we used to pass benchmarker
  638. as the argument to the setup.sh files"""
  639. try:
  640. x = getattr(self.benchmarker, name)
  641. except AttributeError:
  642. print "AttributeError: %s not a member of FrameworkTest or Benchmarker" % name
  643. print "This is probably a bug"
  644. raise
  645. return x
  646. ##############################################################
  647. # Begin __calculate_average_stats
  648. # We have a large amount of raw data for the statistics that
  649. # may be useful for the stats nerds, but most people care about
  650. # a couple of numbers. For now, we're only going to supply:
  651. # * Average CPU
  652. # * Average Memory
  653. # * Total network use
  654. # * Total disk use
  655. # More may be added in the future. If they are, please update
  656. # the above list.
  657. # Note: raw_stats is directly from the __parse_stats method.
  658. # Recall that this consists of a dictionary of timestamps,
  659. # each of which contain a dictionary of stat categories which
  660. # contain a dictionary of stats
  661. ##############################################################
  662. def __calculate_average_stats(self, raw_stats):
  663. raw_stat_collection = dict()
  664. for timestamp, time_dict in raw_stats.items():
  665. for main_header, sub_headers in time_dict.items():
  666. item_to_append = None
  667. if 'cpu' in main_header:
  668. # We want to take the idl stat and subtract it from 100
  669. # to get the time that the CPU is NOT idle.
  670. item_to_append = sub_headers['idl'] - 100.0
  671. elif main_header == 'memory usage':
  672. item_to_append = sub_headers['used']
  673. elif 'net' in main_header:
  674. # Network stats have two parts - recieve and send. We'll use a tuple of
  675. # style (recieve, send)
  676. item_to_append = (sub_headers['recv'], sub_headers['send'])
  677. elif 'dsk' or 'io' in main_header:
  678. # Similar for network, except our tuple looks like (read, write)
  679. item_to_append = (sub_headers['read'], sub_headers['writ'])
  680. if item_to_append is not None:
  681. if main_header not in raw_stat_collection:
  682. raw_stat_collection[main_header] = list()
  683. raw_stat_collection[main_header].append(item_to_append)
  684. # Simple function to determine human readable size
  685. # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
  686. def sizeof_fmt(num):
  687. # We'll assume that any number we get is convertable to a float, just in case
  688. num = float(num)
  689. for x in ['bytes','KB','MB','GB']:
  690. if num < 1024.0 and num > -1024.0:
  691. return "%3.1f%s" % (num, x)
  692. num /= 1024.0
  693. return "%3.1f%s" % (num, 'TB')
  694. # Now we have our raw stats in a readable format - we need to format it for display
  695. # We need a floating point sum, so the built in sum doesn't cut it
  696. display_stat_collection = dict()
  697. for header, values in raw_stat_collection.items():
  698. display_stat = None
  699. if 'cpu' in header:
  700. display_stat = sizeof_fmt(math.fsum(values) / len(values))
  701. elif main_header == 'memory usage':
  702. display_stat = sizeof_fmt(math.fsum(values) / len(values))
  703. elif 'net' in main_header:
  704. receive, send = zip(*values) # unzip
  705. display_stat = {'receive': sizeof_fmt(math.fsum(receive)), 'send': sizeof_fmt(math.fsum(send))}
  706. else: # if 'dsk' or 'io' in header:
  707. read, write = zip(*values) # unzip
  708. display_stat = {'read': sizeof_fmt(math.fsum(read)), 'write': sizeof_fmt(math.fsum(write))}
  709. display_stat_collection[header] = display_stat
  710. return display_stat
  711. ###########################################################################################
  712. # End __calculate_average_stats
  713. #########################################################################################
  714. ##########################################################################################
  715. # Constructor
  716. ##########################################################################################
  717. def __init__(self, name, directory, benchmarker, runTests, args):
  718. self.name = name
  719. self.directory = directory
  720. self.benchmarker = benchmarker
  721. self.runTests = runTests
  722. self.fwroot = benchmarker.fwroot
  723. self.approach = ""
  724. self.classification = ""
  725. self.database = ""
  726. self.framework = ""
  727. self.language = ""
  728. self.orm = ""
  729. self.platform = ""
  730. self.webserver = ""
  731. self.os = ""
  732. self.database_os = ""
  733. self.display_name = ""
  734. self.notes = ""
  735. self.versus = ""
  736. # setup logging
  737. logging.basicConfig(stream=sys.stderr, level=logging.INFO)
  738. self.install_root="%s/%s" % (self.fwroot, "installs")
  739. if benchmarker.install_strategy is 'pertest':
  740. self.install_root="%s/pertest/%s" % (self.install_root, name)
  741. # Used in setup.sh scripts for consistency with
  742. # the bash environment variables
  743. self.troot = self.directory
  744. self.iroot = self.install_root
  745. self.__dict__.update(args)
  746. ############################################################
  747. # End __init__
  748. ############################################################
  749. ############################################################
  750. # End FrameworkTest
  751. ############################################################
  752. # Static methods
  753. def test_order(type_name):
  754. """
  755. This sort ordering is set up specifically to return the length
  756. of the test name. There were SO many problems involved with
  757. 'plaintext' being run first (rather, just not last) that we
  758. needed to ensure that it was run last for every framework.
  759. """
  760. return len(type_name)
  761. def validate_urls(test_name, test_keys):
  762. """
  763. Separated from validate_test because urls are not required anywhere. We know a url is incorrect if it is
  764. empty or does not start with a "/" character. There is no validation done to ensure the url conforms to
  765. the suggested url specifications, although those suggestions are presented if a url fails validation here.
  766. """
  767. example_urls = {
  768. "json_url": "/json",
  769. "db_url": "/mysql/db",
  770. "query_url": "/mysql/queries?queries= or /mysql/queries/",
  771. "fortune_url": "/mysql/fortunes",
  772. "update_url": "/mysql/updates?queries= or /mysql/updates/",
  773. "plaintext_url": "/plaintext"
  774. }
  775. for test_url in ["json_url","db_url","query_url","fortune_url","update_url","plaintext_url"]:
  776. key_value = test_keys.get(test_url, None)
  777. if key_value != None and not key_value.startswith('/'):
  778. errmsg = """`%s` field in test \"%s\" does not appear to be a valid url: \"%s\"\n
  779. Example `%s` url: \"%s\"
  780. """ % (test_url, test_name, key_value, test_url, example_urls[test_url])
  781. raise Exception(errmsg)
  782. def validate_test(test_name, test_keys, directory):
  783. """
  784. Validate benchmark config values for this test based on a schema
  785. """
  786. # Ensure that each FrameworkTest has a framework property, inheriting from top-level if not
  787. if not test_keys['framework']:
  788. test_keys['framework'] = config['framework']
  789. recommended_lang = directory.split('/')[-2]
  790. windows_url = "https://github.com/TechEmpower/FrameworkBenchmarks/milestones/Windows%%20Compatibility"
  791. schema = {
  792. 'language': {
  793. 'help': ('language', 'The language of the framework used, suggestion: %s' % recommended_lang)
  794. },
  795. 'webserver': {
  796. 'help': ('webserver', 'Name of the webserver also referred to as the "front-end server"')
  797. },
  798. 'classification': {
  799. 'allowed': [
  800. ('Fullstack', '...'),
  801. ('Micro', '...'),
  802. ('Platform', '...')
  803. ]
  804. },
  805. 'database': {
  806. 'allowed': [
  807. ('MySQL', 'One of the most popular databases around the web and in TFB'),
  808. ('Postgres', 'An advanced SQL database with a larger feature set than MySQL'),
  809. ('MongoDB', 'A popular document-store database'),
  810. ('Cassandra', 'A highly performant and scalable NoSQL database'),
  811. ('Elasticsearch', 'A distributed RESTful search engine that is used as a database for TFB tests'),
  812. ('Redis', 'An open-sourced, BSD licensed, advanced key-value cache and store'),
  813. ('SQLite', 'A network-less database, still supported for backwards compatibility'),
  814. ('SQLServer', 'Microsoft\'s SQL implementation'),
  815. ('None', 'No database was used for these tests, as is the case with Json Serialization and Plaintext')
  816. ]
  817. },
  818. 'approach': {
  819. 'allowed': [
  820. ('Realistic', '...'),
  821. ('Stripped', '...')
  822. ]
  823. },
  824. 'orm': {
  825. 'allowed': [
  826. ('Full', 'Has a full suite of features like lazy loading, caching, multiple language support, sometimes pre-configured with scripts.'),
  827. ('Micro', 'Has basic database driver capabilities such as establishing a connection and sending queries.'),
  828. ('Raw', 'Tests that do not use an ORM will be classified as "raw" meaning they use the platform\'s raw database connectivity.')
  829. ]
  830. },
  831. 'platform': {
  832. 'help': ('platform', 'Name of the platform this framework runs on, e.g. Node.js, Pypy, hhvm, JRuby ...')
  833. },
  834. 'framework': {
  835. # Guranteed to be here and correct at this point
  836. # key is left here to produce the set of required keys
  837. },
  838. 'os': {
  839. 'allowed': [
  840. ('Linux', 'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'),
  841. ('Windows', 'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s' % windows_url)
  842. ]
  843. },
  844. 'database_os': {
  845. 'allowed': [
  846. ('Linux', 'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'),
  847. ('Windows', 'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s' % windows_url)
  848. ]
  849. }
  850. }
  851. # Confirm required keys are present
  852. required_keys = schema.keys()
  853. missing = list(set(required_keys) - set(test_keys))
  854. if len(missing) > 0:
  855. missingstr = (", ").join(map(str, missing))
  856. raise Exception("benchmark_config.json for test %s is invalid, please amend by adding the following required keys: [%s]"
  857. % (test_name, missingstr))
  858. # Check the (all optional) test urls
  859. validate_urls(test_name, test_keys)
  860. # Check values of keys against schema
  861. for key in required_keys:
  862. val = test_keys.get(key, "").lower()
  863. has_predefined_acceptables = 'allowed' in schema[key]
  864. if has_predefined_acceptables:
  865. allowed = schema[key].get('allowed', [])
  866. acceptable_values, descriptors = zip(*allowed)
  867. acceptable_values = [a.lower() for a in acceptable_values]
  868. if val not in acceptable_values:
  869. msg = ("Invalid `%s` value specified for test \"%s\" in framework \"%s\"; suggestions:\n"
  870. % (key, test_name, test_keys['framework']))
  871. helpinfo = ('\n').join([" `%s` -- %s" % (v, desc) for (v, desc) in zip(acceptable_values, descriptors)])
  872. fullerr = msg + helpinfo + "\n"
  873. raise Exception(fullerr)
  874. elif not has_predefined_acceptables and val == "":
  875. msg = ("Value for `%s` in test \"%s\" in framework \"%s\" was missing:\n"
  876. % (key, test_name, test_keys['framework']))
  877. helpinfo = " %s -- %s" % schema[key]['help']
  878. fullerr = msg + helpinfo + '\n'
  879. raise Exception(fullerr)
  880. def parse_config(config, directory, benchmarker):
  881. """
  882. Parses a config file into a list of FrameworkTest objects
  883. """
  884. tests = []
  885. # The config object can specify multiple tests
  886. # Loop over them and parse each into a FrameworkTest
  887. for test in config['tests']:
  888. tests_to_run = [name for (name,keys) in test.iteritems()]
  889. if "default" not in tests_to_run:
  890. logging.warn("Framework %s does not define a default test in benchmark_config.json", config['framework'])
  891. # Check that each test configuration is acceptable
  892. # Throw exceptions if a field is missing, or how to improve the field
  893. for test_name, test_keys in test.iteritems():
  894. # Validates the benchmark_config entry
  895. validate_test(test_name, test_keys, directory)
  896. # Map test type to a parsed FrameworkTestType object
  897. runTests = dict()
  898. for type_name, type_obj in benchmarker.types.iteritems():
  899. try:
  900. # Makes a FrameWorkTestType object using some of the keys in config
  901. # e.g. JsonTestType uses "json_url"
  902. runTests[type_name] = type_obj.copy().parse(test_keys)
  903. except AttributeError as ae:
  904. # This is quite common - most tests don't support all types
  905. # Quitely log it and move on (debug logging is on in travis and this causes
  906. # ~1500 lines of debug, so I'm totally ignoring it for now
  907. # logging.debug("Missing arguments for test type %s for framework test %s", type_name, test_name)
  908. pass
  909. # We need to sort by test_type to run
  910. sortedTestKeys = sorted(runTests.keys(), key=test_order)
  911. sortedRunTests = OrderedDict()
  912. for sortedTestKey in sortedTestKeys:
  913. sortedRunTests[sortedTestKey] = runTests[sortedTestKey]
  914. # Prefix all test names with framework except 'default' test
  915. # Done at the end so we may still refer to the primary test as `default` in benchmark config error messages
  916. if test_name == 'default':
  917. test_name = config['framework']
  918. else:
  919. test_name = "%s-%s" % (config['framework'], test_name)
  920. # By passing the entire set of keys, each FrameworkTest will have a member for each key
  921. tests.append(FrameworkTest(test_name, directory, benchmarker, sortedRunTests, test_keys))
  922. return tests