framework_test.py 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331
  1. from benchmark.fortune_html_parser import FortuneHTMLParser
  2. from setup.linux import setup_util
  3. import importlib
  4. import os
  5. import subprocess
  6. import time
  7. import re
  8. import pprint
  9. import sys
  10. import traceback
  11. import json
  12. import logging
  13. import csv
  14. import shlex
  15. import math
  16. from utils import header
  17. class FrameworkTest:
  18. ##########################################################################################
  19. # Class variables
  20. ##########################################################################################
  21. headers_template = "-H 'Host: localhost' -H '{accept}' -H 'Connection: keep-alive'"
  22. headers_full_template = "-H 'Host: localhost' -H '{accept}' -H 'Accept-Language: en-US,en;q=0.5' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00' -H 'Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600' -H 'Connection: keep-alive'"
  23. accept_json = "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
  24. accept_html = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
  25. accept_plaintext = "Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
  26. concurrency_template = """
  27. echo ""
  28. echo "---------------------------------------------------------"
  29. echo " Running Primer {name}"
  30. echo " {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
  31. echo "---------------------------------------------------------"
  32. echo ""
  33. {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
  34. sleep 5
  35. echo ""
  36. echo "---------------------------------------------------------"
  37. echo " Running Warmup {name}"
  38. echo " {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}\""
  39. echo "---------------------------------------------------------"
  40. echo ""
  41. {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}"
  42. sleep 5
  43. echo ""
  44. echo "---------------------------------------------------------"
  45. echo " Synchronizing time"
  46. echo "---------------------------------------------------------"
  47. echo ""
  48. ntpdate -s pool.ntp.org
  49. for c in {interval}
  50. do
  51. echo ""
  52. echo "---------------------------------------------------------"
  53. echo " Concurrency: $c for {name}"
  54. echo " {wrk} {headers} -d {duration} -c $c --timeout $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
  55. echo "---------------------------------------------------------"
  56. echo ""
  57. STARTTIME=$(date +"%s")
  58. {wrk} {headers} -d {duration} -c $c --timeout $c -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
  59. echo "STARTTIME $STARTTIME"
  60. echo "ENDTIME $(date +"%s")"
  61. sleep 2
  62. done
  63. """
  64. query_template = """
  65. echo ""
  66. echo "---------------------------------------------------------"
  67. echo " Running Primer {name}"
  68. echo " wrk {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}2\""
  69. echo "---------------------------------------------------------"
  70. echo ""
  71. wrk {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}2"
  72. sleep 5
  73. echo ""
  74. echo "---------------------------------------------------------"
  75. echo " Running Warmup {name}"
  76. echo " wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}2\""
  77. echo "---------------------------------------------------------"
  78. echo ""
  79. wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}2"
  80. sleep 5
  81. echo ""
  82. echo "---------------------------------------------------------"
  83. echo " Synchronizing time"
  84. echo "---------------------------------------------------------"
  85. echo ""
  86. ntpdate -s pool.ntp.org
  87. for c in {interval}
  88. do
  89. echo ""
  90. echo "---------------------------------------------------------"
  91. echo " Queries: $c for {name}"
  92. echo " wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}$c\""
  93. echo "---------------------------------------------------------"
  94. echo ""
  95. STARTTIME=$(date +"%s")
  96. wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}$c"
  97. echo "STARTTIME $STARTTIME"
  98. echo "ENDTIME $(date +"%s")"
  99. sleep 2
  100. done
  101. """
  102. language = None
  103. platform = None
  104. webserver = None
  105. classification = None
  106. database = None
  107. approach = None
  108. orm = None
  109. framework = None
  110. os = None
  111. database_os = None
  112. display_name = None
  113. notes = None
  114. versus = None
  115. ############################################################
  116. # Test Variables
  117. ############################################################
  118. JSON = "json"
  119. DB = "db"
  120. QUERY = "query"
  121. FORTUNE = "fortune"
  122. UPDATE = "update"
  123. PLAINTEXT = "plaintext"
  124. ##########################################################################################
  125. # Public Methods
  126. ##########################################################################################
  127. ############################################################
  128. # Validates the jsonString is a JSON object with a 'message'
  129. # key with the value "hello, world!" (case-insensitive).
  130. ############################################################
  131. def validateJson(self, jsonString, out, err):
  132. err_str = ""
  133. if jsonString is None or len(jsonString) == 0:
  134. err_str += "Empty Response"
  135. return (False, err_str)
  136. try:
  137. obj = {k.lower(): v for k,v in json.loads(jsonString).iteritems()}
  138. if "message" not in obj:
  139. err_str += "Expected key 'message' to be in JSON string "
  140. if obj["message"].lower() != "hello, world!":
  141. err_str += "Message was '{message}', should have been 'Hello, World!' ".format(message=obj["message"])
  142. except:
  143. err_str += "Got exception when trying to validate the JSON test: {exception}".format(exception=traceback.format_exc())
  144. return (True, ) if len(err_str) == 0 else (False, err_str)
  145. ############################################################
  146. # Validates the jsonString is a JSON object that has an "id"
  147. # and a "randomNumber" key, and that both keys map to
  148. # integers.
  149. ############################################################
  150. def validateDb(self, jsonString, out, err):
  151. err_str = ""
  152. if jsonString is None or len(jsonString) == 0:
  153. err_str += "Empty Response"
  154. return (False, err_str)
  155. try:
  156. obj = {k.lower(): v for k,v in json.loads(jsonString).iteritems()}
  157. # We are allowing the single-object array for the DB
  158. # test for now, but will likely remove this later.
  159. if type(obj) == list:
  160. obj = obj[0]
  161. if "id" not in obj or "randomnumber" not in obj:
  162. err_str += "Expected keys id and randomNumber to be in JSON string. "
  163. return (False, err_str)
  164. # This will error out of the value could not parsed to a
  165. # float (this will work with ints, but it will turn them
  166. # into their float equivalent; i.e. "123" => 123.0)
  167. id_ret_val = True
  168. try:
  169. if not isinstance(float(obj["id"]), float):
  170. id_ret_val=False
  171. except:
  172. id_ret_val=False
  173. if not id_ret_val:
  174. err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
  175. random_num_ret_val = True
  176. try:
  177. if not isinstance(float(obj["randomnumber"]), float):
  178. random_num_ret_val=False
  179. except:
  180. random_num_ret_val=False
  181. if not random_num_ret_val:
  182. err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
  183. except:
  184. err_str += "Got exception when trying to validate the db test: {exception}".format(exception=traceback.format_exc())
  185. return (True, ) if len(err_str) == 0 else (False, err_str)
  186. def validateDbStrict(self, jsonString, out, err):
  187. err_str = ""
  188. if jsonString is None or len(jsonString) == 0:
  189. err_str += "Empty Response "
  190. return (False, err_str)
  191. try:
  192. obj = {k.lower(): v for k,v in json.loads(jsonString).iteritems()}
  193. # This will error out of the value could not parsed to a
  194. # float (this will work with ints, but it will turn them
  195. # into their float equivalent; i.e. "123" => 123.0)
  196. id_ret_val = True
  197. try:
  198. if not isinstance(float(obj["id"]), float):
  199. id_ret_val=False
  200. except:
  201. id_ret_val=False
  202. if not id_ret_val:
  203. err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
  204. random_num_ret_val = True
  205. try:
  206. if not isinstance(float(obj["randomnumber"]), float):
  207. random_num_ret_val=False
  208. except:
  209. random_num_ret_val=False
  210. if not random_num_ret_val:
  211. err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
  212. return id_ret_val and random_num_ret_val
  213. except:
  214. err_str += "Got exception when trying to validate the db test: {exception}".format(exception=traceback.format_exc())
  215. return (True, ) if len(err_str) == 0 else (False, err_str)
  216. ############################################################
  217. # Validates the jsonString is an array with a length of
  218. # 2, that each entry in the array is a JSON object, that
  219. # each object has an "id" and a "randomNumber" key, and that
  220. # both keys map to integers.
  221. ############################################################
  222. def validateQuery(self, jsonString, out, err):
  223. err_str = ""
  224. if jsonString is None or len(jsonString) == 0:
  225. err_str += "Empty Response"
  226. return (False, err_str)
  227. try:
  228. arr = [{k.lower(): v for k,v in d.iteritems()} for d in json.loads(jsonString)]
  229. if len(arr) != 2:
  230. err_str += "Expected array of length 2. Got length {length}. ".format(length=len(arr))
  231. for obj in arr:
  232. id_ret_val = True
  233. random_num_ret_val = True
  234. if "id" not in obj or "randomnumber" not in obj:
  235. err_str += "Expected keys id and randomNumber to be in JSON string. "
  236. break
  237. try:
  238. if not isinstance(float(obj["id"]), float):
  239. id_ret_val=False
  240. except:
  241. id_ret_val=False
  242. if not id_ret_val:
  243. err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
  244. try:
  245. if not isinstance(float(obj["randomnumber"]), float):
  246. random_num_ret_val=False
  247. except:
  248. random_num_ret_val=False
  249. if not random_num_ret_val:
  250. err_str += "Expected randomNumber to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
  251. except:
  252. err_str += "Got exception when trying to validate the query test: {exception}".format(exception=traceback.format_exc())
  253. return (True, ) if len(err_str) == 0 else (False, err_str)
  254. ############################################################
  255. # Validates the jsonString is an array with a length of
  256. # 1, that each entry in the array is a JSON object, that
  257. # each object has an "id" and a "randomNumber" key, and that
  258. # both keys map to integers.
  259. ############################################################
  260. def validateQueryOneOrLess(self, jsonString, out, err):
  261. err_str = ""
  262. if jsonString is None or len(jsonString) == 0:
  263. err_str += "Empty Response"
  264. else:
  265. try:
  266. json_load = json.loads(jsonString)
  267. if not isinstance(json_load, list):
  268. err_str += "Expected JSON array, got {typeObj}. ".format(typeObj=type(json_load))
  269. if len(json_load) != 1:
  270. err_str += "Expected array of length 1. Got length {length}. ".format(length=len(json_load))
  271. obj = {k.lower(): v for k,v in json_load[0].iteritems()}
  272. id_ret_val = True
  273. random_num_ret_val = True
  274. if "id" not in obj or "randomnumber" not in obj:
  275. err_str += "Expected keys id and randomNumber to be in JSON string. "
  276. try:
  277. if not isinstance(float(obj["id"]), float):
  278. id_ret_val=False
  279. except:
  280. id_ret_val=False
  281. if not id_ret_val:
  282. err_str += "Expected id to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
  283. try:
  284. if not isinstance(float(obj["randomnumber"]), float):
  285. random_num_ret_val=False
  286. except:
  287. random_num_ret_val=False
  288. if not random_num_ret_val:
  289. err_str += "Expected randomNumber to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
  290. except:
  291. err_str += "Got exception when trying to validate the query test: {exception} ".format(exception=traceback.format_exc())
  292. return (True, ) if len(err_str) == 0 else (False, err_str)
  293. ############################################################
  294. # Validates the jsonString is an array with a length of
  295. # 500, that each entry in the array is a JSON object, that
  296. # each object has an "id" and a "randomNumber" key, and that
  297. # both keys map to integers.
  298. ############################################################
  299. def validateQueryFiveHundredOrMore(self, jsonString, out, err):
  300. err_str = ""
  301. if jsonString is None or len(jsonString) == 0:
  302. err_str += "Empty Response"
  303. return (False, err_str)
  304. try:
  305. arr = [{k.lower(): v for k,v in d.iteritems()} for d in json.loads(jsonString)]
  306. if len(arr) != 500:
  307. err_str += "Expected array of length 500. Got length {length}. ".format(length=len(arr))
  308. return (False, err_str)
  309. for obj in arr:
  310. id_ret_val = True
  311. random_num_ret_val = True
  312. if "id" not in obj or "randomnumber" not in obj:
  313. err_str += "Expected keys id and randomNumber to be in JSON string. "
  314. break
  315. try:
  316. if not isinstance(float(obj["id"]), float):
  317. id_ret_val=False
  318. except:
  319. id_ret_val=False
  320. if not id_ret_val:
  321. err_str += "Expected id to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
  322. try:
  323. if not isinstance(float(obj["randomnumber"]), float):
  324. random_num_ret_val=False
  325. except:
  326. random_num_ret_val=False
  327. if not random_num_ret_val:
  328. err_str += "Expected randomNumber to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
  329. except:
  330. err_str += "Got exception when trying to validate the query test: {exception} ".format(exception=traceback.format_exc())
  331. return (True, ) if len(err_str) == 0 else (False, err_str)
  332. ############################################################
  333. # Parses the given HTML string and asks a FortuneHTMLParser
  334. # whether the parsed string is a valid fortune return.
  335. ############################################################
  336. def validateFortune(self, htmlString, out, err):
  337. err_str = ""
  338. if htmlString is None or len(htmlString) == 0:
  339. err_str += "Empty Response"
  340. return (False, err_str)
  341. try:
  342. parser = FortuneHTMLParser()
  343. parser.feed(htmlString)
  344. return parser.isValidFortune(out)
  345. except:
  346. print "Got exception when trying to validate the fortune test: {exception} ".format(exception=traceback.format_exc())
  347. return (False, err_str)
  348. ############################################################
  349. # Validates the jsonString is an array with a length of
  350. # 2, that each entry in the array is a JSON object, that
  351. # each object has an "id" and a "randomNumber" key, and that
  352. # both keys map to integers.
  353. ############################################################
  354. def validateUpdate(self, jsonString, out, err):
  355. err_str = ""
  356. if jsonString is None or len(jsonString) == 0:
  357. err_str += "Empty Response"
  358. return (False, err_str)
  359. try:
  360. arr = [{k.lower(): v for k,v in d.iteritems()} for d in json.loads(jsonString)]
  361. if len(arr) != 2:
  362. err_str += "Expected array of length 2. Got length {length}.\n".format(length=len(arr))
  363. for obj in arr:
  364. id_ret_val = True
  365. random_num_ret_val = True
  366. if "id" not in obj or "randomnumber" not in obj:
  367. err_str += "Expected keys id and randomNumber to be in JSON string.\n"
  368. return (False, err_str)
  369. try:
  370. if not isinstance(float(obj["id"]), float):
  371. id_ret_val=False
  372. except:
  373. id_ret_val=False
  374. if not id_ret_val:
  375. err_str += "Expected id to be type int or float, got '{rand}'.\n".format(rand=obj["randomnumber"])
  376. try:
  377. if not isinstance(float(obj["randomnumber"]), float):
  378. random_num_ret_val=False
  379. except:
  380. random_num_ret_val=False
  381. if not random_num_ret_val:
  382. err_str += "Expected randomNumber to be type int or float, got '{rand}'.\n".format(rand=obj["randomnumber"])
  383. except:
  384. err_str += "Got exception when trying to validate the update test: {exception}\n".format(exception=traceback.format_exc())
  385. return (True, ) if len(err_str) == 0 else (False, err_str)
  386. ############################################################
  387. #
  388. ############################################################
  389. def validatePlaintext(self, jsonString, out, err):
  390. err_str = ""
  391. if jsonString is None or len(jsonString) == 0:
  392. err_str += "Empty Response"
  393. return (False, err_str)
  394. try:
  395. if not jsonString.lower().strip() == "hello, world!":
  396. err_str += "Expected 'Hello, World!', got '{message}'.\n".format(message=jsonString.strip())
  397. except:
  398. err_str += "Got exception when trying to validate the plaintext test: {exception}\n".format(exception=traceback.format_exc())
  399. return (True, ) if len(err_str) == 0 else (False, err_str)
  400. ############################################################
  401. # start(benchmarker)
  402. # Start the test using it's setup file
  403. ############################################################
  404. def start(self, out, err):
  405. # Load profile for this installation
  406. profile="%s/bash_profile.sh" % self.directory
  407. if not os.path.exists(profile):
  408. logging.warning("Directory %s does not have a bash_profile.sh" % self.directory)
  409. profile="$FWROOT/config/benchmark_profile"
  410. # Setup variables for TROOT and IROOT
  411. self.troot = "%s/%s" % (self.fwroot, self.directory)
  412. self.iroot = self.install_root
  413. setup_util.replace_environ(config=profile,
  414. command='export TROOT=$FWROOT/%s && export IROOT=%s' %
  415. (self.directory, self.install_root))
  416. # Run the module start (inside parent of TROOT)
  417. # - we use the parent as a historical accident - a lot of tests
  418. # use subprocess's cwd argument already
  419. previousDir = os.getcwd()
  420. os.chdir(os.path.dirname(self.troot))
  421. logging.warning("Running setup module (cwd=%s)", os.path.dirname(self.troot))
  422. retcode = self.setup_module.start(self, out, err)
  423. os.chdir(previousDir)
  424. return retcode
  425. ############################################################
  426. # End start
  427. ############################################################
  428. ############################################################
  429. # stop(benchmarker)
  430. # Stops the test using it's setup file
  431. ############################################################
  432. def stop(self, out, err):
  433. # Load profile for this installation
  434. profile="%s/bash_profile.sh" % self.directory
  435. if not os.path.exists(profile):
  436. logging.warning("Directory %s does not have a bash_profile.sh" % self.directory)
  437. profile="$FWROOT/config/benchmark_profile"
  438. setup_util.replace_environ(config=profile,
  439. command='export TROOT=$FWROOT/%s && export IROOT=%s' %
  440. (self.directory, self.install_root))
  441. # Run the module stop (inside parent of TROOT)
  442. # - we use the parent as a historical accident - a lot of tests
  443. # use subprocess's cwd argument already
  444. previousDir = os.getcwd()
  445. os.chdir(os.path.dirname(self.troot))
  446. logging.warning("Running setup module (cwd=%s)", os.path.dirname(self.troot))
  447. os.chdir(self.troot)
  448. retcode = self.setup_module.stop(out, err)
  449. os.chdir(previousDir)
  450. return retcode
  451. ############################################################
  452. # End stop
  453. ############################################################
  454. ############################################################
  455. # verify_urls
  456. # Verifys each of the URLs for this test. THis will sinply
  457. # curl the URL and check for it's return status.
  458. # For each url, a flag will be set on this object for whether
  459. # or not it passed
  460. # Returns True if all verifications succeeded
  461. ############################################################
  462. def verify_urls(self, out, err):
  463. result = True
  464. # JSON
  465. if self.runTests[self.JSON]:
  466. out.write(header("VERIFYING JSON (%s)" % self.json_url))
  467. out.flush()
  468. url = self.benchmarker.generate_url(self.json_url, self.port)
  469. output = self.__curl_url(url, self.JSON, out, err)
  470. out.write("VALIDATING JSON ... ")
  471. ret_tuple = self.validateJson(output, out, err)
  472. if ret_tuple[0]:
  473. self.json_url_passed = True
  474. out.write("PASS\n\n")
  475. else:
  476. self.json_url_passed = False
  477. out.write("\nFAIL" + ret_tuple[1] + "\n\n")
  478. result = False
  479. out.flush()
  480. # DB
  481. if self.runTests[self.DB]:
  482. out.write(header("VERIFYING DB (%s)" % self.db_url))
  483. out.flush()
  484. url = self.benchmarker.generate_url(self.db_url, self.port)
  485. output = self.__curl_url(url, self.DB, out, err)
  486. validate_ret_tuple = self.validateDb(output, out, err)
  487. validate_strict_ret_tuple = self.validateDbStrict(output, out, err)
  488. if validate_ret_tuple[0]:
  489. self.db_url_passed = True
  490. else:
  491. self.db_url_passed = False
  492. if validate_strict_ret_tuple:
  493. self.db_url_warn = False
  494. else:
  495. self.db_url_warn = True
  496. out.write("VALIDATING DB ... ")
  497. if self.db_url_passed:
  498. out.write("PASS")
  499. if self.db_url_warn:
  500. out.write(" (with warnings) " + validate_strict_ret_tuple[1])
  501. out.write("\n\n")
  502. else:
  503. out.write("\nFAIL" + validate_ret_tuple[1])
  504. result = False
  505. out.flush()
  506. # Query
  507. if self.runTests[self.QUERY]:
  508. out.write(header("VERIFYING QUERY (%s)" % self.query_url+"2"))
  509. out.flush()
  510. url = self.benchmarker.generate_url(self.query_url + "2", self.port)
  511. output = self.__curl_url(url, self.QUERY, out, err)
  512. ret_tuple = self.validateQuery(output, out, err)
  513. if ret_tuple[0]:
  514. self.query_url_passed = True
  515. out.write(self.query_url + "2 - PASS\n\n")
  516. else:
  517. self.query_url_passed = False
  518. out.write(self.query_url + "2 - FAIL " + ret_tuple[1] + "\n\n")
  519. out.write("-----------------------------------------------------\n\n")
  520. out.flush()
  521. self.query_url_warn = False
  522. url2 = self.benchmarker.generate_url(self.query_url + "0", self.port)
  523. output2 = self.__curl_url(url2, self.QUERY, out, err)
  524. ret_tuple = self.validateQueryOneOrLess(output2, out, err)
  525. if not ret_tuple[0]:
  526. self.query_url_warn = True
  527. out.write(self.query_url + "0 - WARNING " + ret_tuple[1] + "\n\n")
  528. else:
  529. out.write(self.query_url + "0 - PASS\n\n")
  530. out.write("-----------------------------------------------------\n\n")
  531. out.flush()
  532. url3 = self.benchmarker.generate_url(self.query_url + "foo", self.port)
  533. output3 = self.__curl_url(url3, self.QUERY, out, err)
  534. ret_tuple = self.validateQueryOneOrLess(output3, out, err)
  535. if not ret_tuple[0]:
  536. self.query_url_warn = True
  537. out.write(self.query_url + "foo - WARNING " + ret_tuple[1] + "\n\n")
  538. else:
  539. out.write(self.query_url + "foo - PASS\n\n")
  540. out.write("-----------------------------------------------------\n\n")
  541. out.flush()
  542. url4 = self.benchmarker.generate_url(self.query_url + "501", self.port)
  543. output4 = self.__curl_url(url4, self.QUERY, out, err)
  544. ret_tuple = self.validateQueryFiveHundredOrMore(output4, out, err)
  545. if not ret_tuple[0]:
  546. self.query_url_warn = True
  547. out.write(self.query_url + "501 - WARNING " + ret_tuple[1] + "\n\n")
  548. else:
  549. out.write(self.query_url + "501 - PASS\n\n")
  550. out.write("-----------------------------------------------------\n\n\n")
  551. out.flush()
  552. out.write("VALIDATING QUERY ... ")
  553. if self.query_url_passed:
  554. out.write("PASS")
  555. if self.query_url_warn:
  556. out.write(" (with warnings)")
  557. out.write("\n\n")
  558. else:
  559. out.write("\nFAIL " + ret_tuple[1] + "\n\n")
  560. result = False
  561. out.flush()
  562. # Fortune
  563. if self.runTests[self.FORTUNE]:
  564. out.write(header("VERIFYING FORTUNE (%s)" % self.fortune_url))
  565. out.flush()
  566. url = self.benchmarker.generate_url(self.fortune_url, self.port)
  567. output = self.__curl_url(url, self.FORTUNE, out, err)
  568. out.write("VALIDATING FORTUNE ... ")
  569. if self.validateFortune(output, out, err):
  570. self.fortune_url_passed = True
  571. out.write("PASS\n\n")
  572. else:
  573. self.fortune_url_passed = False
  574. out.write("\nFAIL\n\n")
  575. result = False
  576. out.flush()
  577. # Update
  578. if self.runTests[self.UPDATE]:
  579. out.write(header("VERIFYING UPDATE (%s)" % self.update_url))
  580. out.flush()
  581. url = self.benchmarker.generate_url(self.update_url + "2", self.port)
  582. output = self.__curl_url(url, self.UPDATE, out, err)
  583. out.write("VALIDATING UPDATE ... ")
  584. ret_tuple = self.validateUpdate(output, out, err)
  585. if ret_tuple[0]:
  586. self.update_url_passed = True
  587. out.write("PASS\n\n")
  588. else:
  589. self.update_url_passed = False
  590. out.write("\nFAIL " + ret_tuple[1] + "\n\n")
  591. result = False
  592. out.flush()
  593. # plaintext
  594. if self.runTests[self.PLAINTEXT]:
  595. out.write(header("VERIFYING PLAINTEXT (%s)" % self.plaintext_url))
  596. out.flush()
  597. url = self.benchmarker.generate_url(self.plaintext_url, self.port)
  598. output = self.__curl_url(url, self.PLAINTEXT, out, err)
  599. out.write("VALIDATING PLAINTEXT ... ")
  600. ret_tuple = self.validatePlaintext(output, out, err)
  601. if ret_tuple[0]:
  602. self.plaintext_url_passed = True
  603. out.write("PASS\n\n")
  604. else:
  605. self.plaintext_url_passed = False
  606. out.write("\nFAIL\n\n" + ret_tuple[1] + "\n\n")
  607. result = False
  608. out.flush()
  609. return result
  610. ############################################################
  611. # End verify_urls
  612. ############################################################
  613. ############################################################
  614. # contains_type(type)
  615. # true if this test contains an implementation of the given
  616. # test type (json, db, etc.)
  617. ############################################################
  618. def contains_type(self, type):
  619. try:
  620. if type == self.JSON and self.json_url is not None:
  621. return True
  622. if type == self.DB and self.db_url is not None:
  623. return True
  624. if type == self.QUERY and self.query_url is not None:
  625. return True
  626. if type == self.FORTUNE and self.fortune_url is not None:
  627. return True
  628. if type == self.UPDATE and self.update_url is not None:
  629. return True
  630. if type == self.PLAINTEXT and self.plaintext_url is not None:
  631. return True
  632. except AttributeError:
  633. pass
  634. return False
  635. ############################################################
  636. # End stop
  637. ############################################################
  638. ############################################################
  639. # benchmark
  640. # Runs the benchmark for each type of test that it implements
  641. # JSON/DB/Query.
  642. ############################################################
  643. def benchmark(self, out, err):
  644. # JSON
  645. if self.runTests[self.JSON]:
  646. try:
  647. out.write("BENCHMARKING JSON ... ")
  648. out.flush()
  649. results = None
  650. output_file = self.benchmarker.output_file(self.name, self.JSON)
  651. if not os.path.exists(output_file):
  652. with open(output_file, 'w'):
  653. # Simply opening the file in write mode should create the empty file.
  654. pass
  655. if self.json_url_passed:
  656. remote_script = self.__generate_concurrency_script(self.json_url, self.port, self.accept_json)
  657. self.__begin_logging(self.JSON)
  658. self.__run_benchmark(remote_script, output_file, err)
  659. self.__end_logging()
  660. results = self.__parse_test(self.JSON)
  661. print results
  662. self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'])
  663. out.write( "Complete\n" )
  664. out.flush()
  665. except AttributeError:
  666. pass
  667. # DB
  668. if self.runTests[self.DB]:
  669. try:
  670. out.write("BENCHMARKING DB ... ")
  671. out.flush()
  672. results = None
  673. output_file = self.benchmarker.output_file(self.name, self.DB)
  674. warning_file = self.benchmarker.warning_file(self.name, self.DB)
  675. if not os.path.exists(output_file):
  676. with open(output_file, 'w'):
  677. # Simply opening the file in write mode should create the empty file.
  678. pass
  679. if self.db_url_warn:
  680. with open(warning_file, 'w'):
  681. pass
  682. if self.db_url_passed:
  683. remote_script = self.__generate_concurrency_script(self.db_url, self.port, self.accept_json)
  684. self.__begin_logging(self.DB)
  685. self.__run_benchmark(remote_script, output_file, err)
  686. self.__end_logging()
  687. results = self.__parse_test(self.DB)
  688. self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'])
  689. out.write( "Complete\n" )
  690. except AttributeError:
  691. pass
  692. # Query
  693. if self.runTests[self.QUERY]:
  694. try:
  695. out.write("BENCHMARKING Query ... ")
  696. out.flush()
  697. results = None
  698. output_file = self.benchmarker.output_file(self.name, self.QUERY)
  699. warning_file = self.benchmarker.warning_file(self.name, self.QUERY)
  700. if not os.path.exists(output_file):
  701. with open(output_file, 'w'):
  702. # Simply opening the file in write mode should create the empty file.
  703. pass
  704. if self.query_url_warn:
  705. with open(warning_file, 'w'):
  706. pass
  707. if self.query_url_passed:
  708. remote_script = self.__generate_query_script(self.query_url, self.port, self.accept_json)
  709. self.__begin_logging(self.QUERY)
  710. self.__run_benchmark(remote_script, output_file, err)
  711. self.__end_logging()
  712. results = self.__parse_test(self.QUERY)
  713. self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'])
  714. out.write( "Complete\n" )
  715. out.flush()
  716. except AttributeError:
  717. pass
  718. # fortune
  719. if self.runTests[self.FORTUNE]:
  720. try:
  721. out.write("BENCHMARKING Fortune ... ")
  722. out.flush()
  723. results = None
  724. output_file = self.benchmarker.output_file(self.name, self.FORTUNE)
  725. if not os.path.exists(output_file):
  726. with open(output_file, 'w'):
  727. # Simply opening the file in write mode should create the empty file.
  728. pass
  729. if self.fortune_url_passed:
  730. remote_script = self.__generate_concurrency_script(self.fortune_url, self.port, self.accept_html)
  731. self.__begin_logging(self.FORTUNE)
  732. self.__run_benchmark(remote_script, output_file, err)
  733. self.__end_logging()
  734. results = self.__parse_test(self.FORTUNE)
  735. self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'])
  736. out.write( "Complete\n" )
  737. out.flush()
  738. except AttributeError:
  739. pass
  740. # update
  741. if self.runTests[self.UPDATE]:
  742. try:
  743. out.write("BENCHMARKING Update ... ")
  744. out.flush()
  745. results = None
  746. output_file = self.benchmarker.output_file(self.name, self.UPDATE)
  747. if not os.path.exists(output_file):
  748. with open(output_file, 'w'):
  749. # Simply opening the file in write mode should create the empty file.
  750. pass
  751. if self.update_url_passed:
  752. remote_script = self.__generate_query_script(self.update_url, self.port, self.accept_json)
  753. self.__begin_logging(self.UPDATE)
  754. self.__run_benchmark(remote_script, output_file, err)
  755. self.__end_logging()
  756. results = self.__parse_test(self.UPDATE)
  757. self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'])
  758. out.write( "Complete\n" )
  759. out.flush()
  760. except AttributeError:
  761. pass
  762. # plaintext
  763. if self.runTests[self.PLAINTEXT]:
  764. try:
  765. out.write("BENCHMARKING Plaintext ... ")
  766. out.flush()
  767. results = None
  768. output_file = self.benchmarker.output_file(self.name, self.PLAINTEXT)
  769. if not os.path.exists(output_file):
  770. with open(output_file, 'w'):
  771. # Simply opening the file in write mode should create the empty file.
  772. pass
  773. if self.plaintext_url_passed:
  774. remote_script = self.__generate_concurrency_script(self.plaintext_url, self.port, self.accept_plaintext, wrk_command="wrk", intervals=[256,1024,4096,16384], pipeline="16")
  775. self.__begin_logging(self.PLAINTEXT)
  776. self.__run_benchmark(remote_script, output_file, err)
  777. self.__end_logging()
  778. results = self.__parse_test(self.PLAINTEXT)
  779. self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])
  780. out.write( "Complete\n" )
  781. out.flush()
  782. except AttributeError:
  783. traceback.print_exc()
  784. pass
  785. ############################################################
  786. # End benchmark
  787. ############################################################
  788. ############################################################
  789. # parse_all
  790. # Method meant to be run for a given timestamp
  791. ############################################################
  792. def parse_all(self):
  793. # JSON
  794. if os.path.exists(self.benchmarker.get_output_file(self.name, self.JSON)):
  795. results = self.__parse_test(self.JSON)
  796. self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'])
  797. # DB
  798. if os.path.exists(self.benchmarker.get_output_file(self.name, self.DB)):
  799. results = self.__parse_test(self.DB)
  800. self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'])
  801. # Query
  802. if os.path.exists(self.benchmarker.get_output_file(self.name, self.QUERY)):
  803. results = self.__parse_test(self.QUERY)
  804. self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'])
  805. # Fortune
  806. if os.path.exists(self.benchmarker.get_output_file(self.name, self.FORTUNE)):
  807. results = self.__parse_test(self.FORTUNE)
  808. self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'])
  809. # Update
  810. if os.path.exists(self.benchmarker.get_output_file(self.name, self.UPDATE)):
  811. results = self.__parse_test(self.UPDATE)
  812. self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'])
  813. # Plaintext
  814. if os.path.exists(self.benchmarker.get_output_file(self.name, self.PLAINTEXT)):
  815. results = self.__parse_test(self.PLAINTEXT)
  816. self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])
  817. ############################################################
  818. # End parse_all
  819. ############################################################
  820. ############################################################
  821. # __parse_test(test_type)
  822. ############################################################
  823. def __parse_test(self, test_type):
  824. try:
  825. results = dict()
  826. results['results'] = []
  827. stats = []
  828. if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
  829. with open(self.benchmarker.output_file(self.name, test_type)) as raw_data:
  830. is_warmup = True
  831. rawData = None
  832. for line in raw_data:
  833. if "Queries:" in line or "Concurrency:" in line:
  834. is_warmup = False
  835. rawData = None
  836. continue
  837. if "Warmup" in line or "Primer" in line:
  838. is_warmup = True
  839. continue
  840. if not is_warmup:
  841. if rawData == None:
  842. rawData = dict()
  843. results['results'].append(rawData)
  844. #if "Requests/sec:" in line:
  845. # m = re.search("Requests/sec:\s+([0-9]+)", line)
  846. # rawData['reportedResults'] = m.group(1)
  847. # search for weighttp data such as succeeded and failed.
  848. if "Latency" in line:
  849. m = re.findall("([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
  850. if len(m) == 4:
  851. rawData['latencyAvg'] = m[0]
  852. rawData['latencyStdev'] = m[1]
  853. rawData['latencyMax'] = m[2]
  854. # rawData['latencyStdevPercent'] = m[3]
  855. #if "Req/Sec" in line:
  856. # m = re.findall("([0-9]+\.*[0-9]*[k|%]*)", line)
  857. # if len(m) == 4:
  858. # rawData['requestsAvg'] = m[0]
  859. # rawData['requestsStdev'] = m[1]
  860. # rawData['requestsMax'] = m[2]
  861. # rawData['requestsStdevPercent'] = m[3]
  862. #if "requests in" in line:
  863. # m = re.search("requests in ([0-9]+\.*[0-9]*[ms|s|m|h]+)", line)
  864. # if m != None:
  865. # # parse out the raw time, which may be in minutes or seconds
  866. # raw_time = m.group(1)
  867. # if "ms" in raw_time:
  868. # rawData['total_time'] = float(raw_time[:len(raw_time)-2]) / 1000.0
  869. # elif "s" in raw_time:
  870. # rawData['total_time'] = float(raw_time[:len(raw_time)-1])
  871. # elif "m" in raw_time:
  872. # rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 60.0
  873. # elif "h" in raw_time:
  874. # rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 3600.0
  875. if "requests in" in line:
  876. m = re.search("([0-9]+) requests in", line)
  877. if m != None:
  878. rawData['totalRequests'] = int(m.group(1))
  879. if "Socket errors" in line:
  880. if "connect" in line:
  881. m = re.search("connect ([0-9]+)", line)
  882. rawData['connect'] = int(m.group(1))
  883. if "read" in line:
  884. m = re.search("read ([0-9]+)", line)
  885. rawData['read'] = int(m.group(1))
  886. if "write" in line:
  887. m = re.search("write ([0-9]+)", line)
  888. rawData['write'] = int(m.group(1))
  889. if "timeout" in line:
  890. m = re.search("timeout ([0-9]+)", line)
  891. rawData['timeout'] = int(m.group(1))
  892. if "Non-2xx" in line:
  893. m = re.search("Non-2xx or 3xx responses: ([0-9]+)", line)
  894. if m != None:
  895. rawData['5xx'] = int(m.group(1))
  896. if "STARTTIME" in line:
  897. m = re.search("[0-9]+", line)
  898. rawData["startTime"] = int(m.group(0))
  899. if "ENDTIME" in line:
  900. m = re.search("[0-9]+", line)
  901. rawData["endTime"] = int(m.group(0))
  902. test_stats = self.__parse_stats(test_type, rawData["startTime"], rawData["endTime"], 1)
  903. # rawData["averageStats"] = self.__calculate_average_stats(test_stats)
  904. stats.append(test_stats)
  905. with open(self.benchmarker.stats_file(self.name, test_type) + ".json", "w") as stats_file:
  906. json.dump(stats, stats_file)
  907. return results
  908. except IOError:
  909. return None
  910. ############################################################
  911. # End benchmark
  912. ############################################################
  913. ##########################################################################################
  914. # Private Methods
  915. ##########################################################################################
  916. ############################################################
  917. # __run_benchmark(script, output_file)
  918. # Runs a single benchmark using the script which is a bash
  919. # template that uses weighttp to run the test. All the results
  920. # outputed to the output_file.
  921. ############################################################
  922. def __run_benchmark(self, script, output_file, err):
  923. with open(output_file, 'w') as raw_file:
  924. p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" "), stdin=subprocess.PIPE, stdout=raw_file, stderr=err)
  925. p.communicate(script)
  926. err.flush()
  927. ############################################################
  928. # End __run_benchmark
  929. ############################################################
  930. ############################################################
  931. # __generate_concurrency_script(url, port)
  932. # Generates the string containing the bash script that will
  933. # be run on the client to benchmark a single test. This
  934. # specifically works for the variable concurrency tests (JSON
  935. # and DB)
  936. ############################################################
  937. def __generate_concurrency_script(self, url, port, accept_header, wrk_command="wrk", intervals=[], pipeline=""):
  938. if len(intervals) == 0:
  939. intervals = self.benchmarker.concurrency_levels
  940. headers = self.__get_request_headers(accept_header)
  941. return self.concurrency_template.format(max_concurrency=self.benchmarker.max_concurrency,
  942. max_threads=self.benchmarker.max_threads, name=self.name, duration=self.benchmarker.duration,
  943. interval=" ".join("{}".format(item) for item in intervals),
  944. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command,
  945. pipeline=pipeline)
  946. ############################################################
  947. # End __generate_concurrency_script
  948. ############################################################
  949. ############################################################
  950. # __generate_query_script(url, port)
  951. # Generates the string containing the bash script that will
  952. # be run on the client to benchmark a single test. This
  953. # specifically works for the variable query tests (Query)
  954. ############################################################
  955. def __generate_query_script(self, url, port, accept_header):
  956. headers = self.__get_request_headers(accept_header)
  957. return self.query_template.format(max_concurrency=self.benchmarker.max_concurrency,
  958. max_threads=self.benchmarker.max_threads, name=self.name, duration=self.benchmarker.duration,
  959. interval=" ".join("{}".format(item) for item in self.benchmarker.query_intervals),
  960. server_host=self.benchmarker.server_host, port=port, url=url, headers=headers)
  961. ############################################################
  962. # End __generate_query_script
  963. ############################################################
  964. ############################################################
  965. # __get_request_headers(accept_header)
  966. # Generates the complete HTTP header string
  967. ############################################################
  968. def __get_request_headers(self, accept_header):
  969. return self.headers_template.format(accept=accept_header)
  970. ############################################################
  971. # End __format_request_headers
  972. ############################################################
  973. ############################################################
  974. # __curl_url
  975. # Dump HTTP response and headers. Throw exception if there
  976. # is an HTTP error.
  977. ############################################################
  978. def __curl_url(self, url, testType, out, err):
  979. output = None
  980. try:
  981. # Use -m 15 to make curl stop trying after 15sec.
  982. # Use -i to output response with headers.
  983. # Don't use -f so that the HTTP response code is ignored.
  984. # Use --stderr - to redirect stderr to stdout so we get
  985. # error output for sure in stdout.
  986. # Use -sS to hide progress bar, but show errors.
  987. subprocess.check_call(["curl", "-m", "15", "-i", "-sS", url], stderr=err, stdout=out)
  988. # HTTP output may not end in a newline, so add that here.
  989. out.write( "\n\n" )
  990. out.flush()
  991. err.flush()
  992. # We need to get the respond body from the curl and return it.
  993. p = subprocess.Popen(["curl", "-m", "15", "-s", url], stdout=subprocess.PIPE)
  994. output = p.communicate()
  995. except:
  996. pass
  997. if output:
  998. # We have the response body - return it
  999. return output[0]
  1000. ##############################################################
  1001. # End __curl_url
  1002. ##############################################################
  1003. def requires_database(self):
  1004. """Returns True/False if this test requires a database"""
  1005. return (self.contains_type(self.FORTUNE) or
  1006. self.contains_type(self.DB) or
  1007. self.contains_type(self.QUERY) or
  1008. self.contains_type(self.UPDATE))
  1009. ############################################################
  1010. # __begin_logging
  1011. # Starts a thread to monitor the resource usage, to be synced with the client's time
  1012. # TODO: MySQL and InnoDB are possible. Figure out how to implement them.
  1013. ############################################################
  1014. def __begin_logging(self, test_name):
  1015. output_file = "{file_name}".format(file_name=self.benchmarker.get_stats_file(self.name, test_name))
  1016. dstat_string = "dstat -afilmprsT --aio --fs --ipc --lock --raw --socket --tcp \
  1017. --raw --socket --tcp --udp --unix --vm --disk-util \
  1018. --rpc --rpcd --output {output_file}".format(output_file=output_file)
  1019. cmd = shlex.split(dstat_string)
  1020. dev_null = open(os.devnull, "w")
  1021. self.subprocess_handle = subprocess.Popen(cmd, stdout=dev_null)
  1022. ##############################################################
  1023. # End __begin_logging
  1024. ##############################################################
  1025. ##############################################################
  1026. # Begin __end_logging
  1027. # Stops the logger thread and blocks until shutdown is complete.
  1028. ##############################################################
  1029. def __end_logging(self):
  1030. self.subprocess_handle.terminate()
  1031. self.subprocess_handle.communicate()
  1032. ##############################################################
  1033. # End __end_logging
  1034. ##############################################################
  1035. ##############################################################
  1036. # Begin __parse_stats
  1037. # For each test type, process all the statistics, and return a multi-layered dictionary
  1038. # that has a structure as follows:
  1039. # (timestamp)
  1040. # | (main header) - group that the stat is in
  1041. # | | (sub header) - title of the stat
  1042. # | | | (stat) - the stat itself, usually a floating point number
  1043. ##############################################################
  1044. def __parse_stats(self, test_type, start_time, end_time, interval):
  1045. stats_dict = dict()
  1046. stats_file = self.benchmarker.stats_file(self.name, test_type)
  1047. with open(stats_file) as stats:
  1048. while(stats.next() != "\n"): # dstat doesn't output a completely compliant CSV file - we need to strip the header
  1049. pass
  1050. stats_reader = csv.reader(stats)
  1051. main_header = stats_reader.next()
  1052. sub_header = stats_reader.next()
  1053. time_row = sub_header.index("epoch")
  1054. int_counter = 0
  1055. for row in stats_reader:
  1056. time = float(row[time_row])
  1057. int_counter+=1
  1058. if time < start_time:
  1059. continue
  1060. elif time > end_time:
  1061. return stats_dict
  1062. if int_counter % interval != 0:
  1063. continue
  1064. row_dict = dict()
  1065. for nextheader in main_header:
  1066. if nextheader != "":
  1067. row_dict[nextheader] = dict()
  1068. header = ""
  1069. for item_num, column in enumerate(row):
  1070. if(len(main_header[item_num]) != 0):
  1071. header = main_header[item_num]
  1072. row_dict[header][sub_header[item_num]] = float(column) # all the stats are numbers, so we want to make sure that they stay that way in json
  1073. stats_dict[time] = row_dict
  1074. return stats_dict
  1075. ##############################################################
  1076. # End __parse_stats
  1077. ##############################################################
  1078. def __getattr__(self, name):
  1079. """For backwards compatibility, we used to pass benchmarker
  1080. as the argument to the setup.py files"""
  1081. return getattr(self.benchmarker, name)
  1082. ##############################################################
  1083. # Begin __calculate_average_stats
  1084. # We have a large amount of raw data for the statistics that
  1085. # may be useful for the stats nerds, but most people care about
  1086. # a couple of numbers. For now, we're only going to supply:
  1087. # * Average CPU
  1088. # * Average Memory
  1089. # * Total network use
  1090. # * Total disk use
  1091. # More may be added in the future. If they are, please update
  1092. # the above list.
  1093. # Note: raw_stats is directly from the __parse_stats method.
  1094. # Recall that this consists of a dictionary of timestamps,
  1095. # each of which contain a dictionary of stat categories which
  1096. # contain a dictionary of stats
  1097. ##############################################################
  1098. def __calculate_average_stats(self, raw_stats):
  1099. raw_stat_collection = dict()
  1100. for timestamp, time_dict in raw_stats.items():
  1101. for main_header, sub_headers in time_dict.items():
  1102. item_to_append = None
  1103. if 'cpu' in main_header:
  1104. # We want to take the idl stat and subtract it from 100
  1105. # to get the time that the CPU is NOT idle.
  1106. item_to_append = sub_headers['idl'] - 100.0
  1107. elif main_header == 'memory usage':
  1108. item_to_append = sub_headers['used']
  1109. elif 'net' in main_header:
  1110. # Network stats have two parts - recieve and send. We'll use a tuple of
  1111. # style (recieve, send)
  1112. item_to_append = (sub_headers['recv'], sub_headers['send'])
  1113. elif 'dsk' or 'io' in main_header:
  1114. # Similar for network, except our tuple looks like (read, write)
  1115. item_to_append = (sub_headers['read'], sub_headers['writ'])
  1116. if item_to_append is not None:
  1117. if main_header not in raw_stat_collection:
  1118. raw_stat_collection[main_header] = list()
  1119. raw_stat_collection[main_header].append(item_to_append)
  1120. # Simple function to determine human readable size
  1121. # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
  1122. def sizeof_fmt(num):
  1123. # We'll assume that any number we get is convertable to a float, just in case
  1124. num = float(num)
  1125. for x in ['bytes','KB','MB','GB']:
  1126. if num < 1024.0 and num > -1024.0:
  1127. return "%3.1f%s" % (num, x)
  1128. num /= 1024.0
  1129. return "%3.1f%s" % (num, 'TB')
  1130. # Now we have our raw stats in a readable format - we need to format it for display
  1131. # We need a floating point sum, so the built in sum doesn't cut it
  1132. display_stat_collection = dict()
  1133. for header, values in raw_stat_collection.items():
  1134. display_stat = None
  1135. if 'cpu' in header:
  1136. display_stat = sizeof_fmt(math.fsum(values) / len(values))
  1137. elif main_header == 'memory usage':
  1138. display_stat = sizeof_fmt(math.fsum(values) / len(values))
  1139. elif 'net' in main_header:
  1140. receive, send = zip(*values) # unzip
  1141. display_stat = {'receive': sizeof_fmt(math.fsum(receive)), 'send': sizeof_fmt(math.fsum(send))}
  1142. else: # if 'dsk' or 'io' in header:
  1143. read, write = zip(*values) # unzip
  1144. display_stat = {'read': sizeof_fmt(math.fsum(read)), 'write': sizeof_fmt(math.fsum(write))}
  1145. display_stat_collection[header] = display_stat
  1146. return display_stat
  1147. ###########################################################################################
  1148. # End __calculate_average_stats
  1149. #########################################################################################
  1150. ##########################################################################################
  1151. # Constructor
  1152. ##########################################################################################
  1153. def __init__(self, name, directory, benchmarker, runTests, args):
  1154. self.name = name
  1155. self.directory = directory
  1156. self.benchmarker = benchmarker
  1157. self.runTests = runTests
  1158. self.fwroot = benchmarker.fwroot
  1159. # setup logging
  1160. logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
  1161. self.install_root="%s/%s" % (self.fwroot, "installs")
  1162. if benchmarker.install_strategy is 'pertest':
  1163. self.install_root="%s/pertest/%s" % (self.install_root, name)
  1164. self.__dict__.update(args)
  1165. # ensure directory has __init__.py file so that we can use it as a Python package
  1166. if not os.path.exists(os.path.join(directory, "__init__.py")):
  1167. open(os.path.join(directory, "__init__.py"), 'w').close()
  1168. self.setup_module = setup_module = importlib.import_module(directory + '.' + self.setup_file)
  1169. ############################################################
  1170. # End __init__
  1171. ############################################################
  1172. ############################################################
  1173. # End FrameworkTest
  1174. ############################################################
  1175. ##########################################################################################
  1176. # Static methods
  1177. ##########################################################################################
  1178. ##############################################################
  1179. # parse_config(config, directory, benchmarker)
  1180. # parses a config file and returns a list of FrameworkTest
  1181. # objects based on that config file.
  1182. ##############################################################
  1183. def parse_config(config, directory, benchmarker):
  1184. tests = []
  1185. # The config object can specify multiple tests, we neep to loop
  1186. # over them and parse them out
  1187. for test in config['tests']:
  1188. for key, value in test.iteritems():
  1189. test_name = config['framework']
  1190. runTests = dict()
  1191. runTests["json"] = (benchmarker.type == "all" or benchmarker.type == "json") and value.get("json_url", False)
  1192. runTests["db"] = (benchmarker.type == "all" or benchmarker.type == "db") and value.get("db_url", False)
  1193. runTests["query"] = (benchmarker.type == "all" or benchmarker.type == "query") and value.get("query_url", False)
  1194. runTests["fortune"] = (benchmarker.type == "all" or benchmarker.type == "fortune") and value.get("fortune_url", False)
  1195. runTests["update"] = (benchmarker.type == "all" or benchmarker.type == "update") and value.get("update_url", False)
  1196. runTests["plaintext"] = (benchmarker.type == "all" or benchmarker.type == "plaintext") and value.get("plaintext_url", False)
  1197. # if the test uses the 'defualt' keywork, then we don't
  1198. # append anything to it's name. All configs should only have 1 default
  1199. if key != 'default':
  1200. # we need to use the key in the test_name
  1201. test_name = test_name + "-" + key
  1202. tests.append(FrameworkTest(test_name, directory, benchmarker, runTests, value))
  1203. return tests
  1204. ##############################################################
  1205. # End parse_config
  1206. ##############################################################