generate_module_docs.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. # This file generates the modules.md content
  2. import os, json, sys, time, fnmatch, re
  3. class ModuleDocGenerator(object):
  4. PATH_GENERATED_MARKDOWN = "../docs/modules.md"
  5. PATH_MODULES_DOCS = "../docs/modules/"
  6. # Contains the output until it should be written to disk
  7. markdown_string = ""
  8. def execute(self, data):
  9. # Validate that we got some methods back. 155 is an arbitrary large number.
  10. if len(data) < 1:
  11. print "ERR: Invalid data"
  12. exit()
  13. functions_parsed = self.parse_function_list(data)
  14. self.output_markdown(functions_parsed)
  15. print "Markdown doc created successfully at " + self.PATH_GENERATED_MARKDOWN
  16. def parse_function_list(self, functions):
  17. data = {}
  18. for elem in functions:
  19. module = elem["module"]
  20. # TODO: What about the pvx module?
  21. if module == "":
  22. module = "core"
  23. if module not in data:
  24. data[module] = []
  25. data[module].append({"name": elem["name"], "return": elem["ret"], "params": elem["params"]})
  26. return data
  27. def output_markdown(self, data):
  28. self.markdown_header()
  29. for key in sorted(data):
  30. methods = data[key]
  31. # Sort the functions by name alphabetically
  32. methods = sorted(methods, key = lambda k: k['name'])
  33. self.markdown_section_heading(key)
  34. self.markdown_section_content(key, methods)
  35. self.markdown_write()
  36. return True
  37. def markdown_header(self):
  38. self.markdown_string += self.read_file_to_string("header.md")
  39. return True
  40. def markdown_section_heading(self, heading):
  41. self.markdown_string += "## " + heading + " ##\n\n"
  42. self.markdown_string += self.read_file_to_string(heading + ".header.md")
  43. return True
  44. def markdown_section_content(self, module, methods):
  45. if module == "core":
  46. module_prefix = ""
  47. else:
  48. module_prefix = module + "."
  49. for value in methods:
  50. self.markdown_string += "#### KSR." + module_prefix + value["name"] + "() ####\n\n"
  51. # Sanitize the return values
  52. if value["return"] == "none":
  53. return_value = "void"
  54. else:
  55. return_value = value["return"]
  56. # Sanitize the parameter values
  57. if value["params"] == "none":
  58. params_value = ""
  59. else:
  60. params_value = value["params"]
  61. # Generate the output string for the markdown page
  62. self.markdown_string += "<a target='_blank' href='/docs/modules/devel/modules/" + module + ".html#" \
  63. + module + ".f." + value["name"] + "'> `" + return_value + " " + value["name"] \
  64. + "(" + params_value + ")` </a>\n\n"
  65. self.markdown_string += self.read_file_to_string(module + "." + value["name"] + ".md")
  66. return True
  67. def markdown_write(self):
  68. f = open(self.PATH_GENERATED_MARKDOWN, "w")
  69. f.write(self.markdown_string)
  70. f.close()
  71. return True
  72. def read_file_to_string(self, filename):
  73. path = self.PATH_MODULES_DOCS + filename
  74. if os.path.isfile(path):
  75. with open(path, 'r') as myfile:
  76. return myfile.read() + "\n"
  77. return ""
  78. class KemiFileExportParser(object):
  79. # These functions are created by a macro so makes the parsing somewhat tricky, for not they are statically defined
  80. macro_functions = {
  81. "t_set_auto_inv_100": "int state",
  82. "t_set_disable_6xx": "int state",
  83. "t_set_disable_failover": "int state",
  84. "t_set_no_e2e_cancel_reason": "int state",
  85. "t_set_disable_internal_reply": "int state"
  86. }
  87. # These files export the KEMI functions in a special way so we map them manually
  88. # TODO: Discuss with @miconda if core/HDR/pv/x should be added as well or not
  89. special_exports = [
  90. #{"filename": "kemi.c", "export": "_sr_kemi_core", "folder": "/core/"},
  91. #{"filename": "kemi.c", "export": "_sr_kemi_hdr", "folder": "/core/"},
  92. #{"filename": "app_lua_mod.c", "export": "sr_kemi_app_lua_rpc_exports", "folder": "/modules/app_lua/"}
  93. ]
  94. def generate_kemi_export_list(self, source_path):
  95. files = self.list_c_files_in_directory(source_path)
  96. lists = []
  97. for file in files:
  98. with open(file) as f:
  99. lines = f.readlines()
  100. export_name = self.find_c_file_kemi_export(file, lines)
  101. if export_name:
  102. export_functions = self.extract_c_file_kemi_export_lines(file, lines, export_name)
  103. lists = lists + export_functions
  104. print "Found ", len(export_functions), "functions", "Total:", len(lists)
  105. # Handle some special files separately
  106. for elem in self.special_exports:
  107. file = source_path + elem["folder"] + elem["filename"]
  108. with open(file) as f:
  109. lines = f.readlines()
  110. lists = lists + self.extract_c_file_kemi_export_lines(file, lines, elem["export"])
  111. return lists
  112. def find_c_file_kemi_export(self, filename, lines):
  113. param = None
  114. for line in lines:
  115. if line.find("sr_kemi_modules_add") >= 0:
  116. line = line.lstrip(" ")
  117. line = line.lstrip("\t")
  118. if line.find("sr_kemi_modules_add") == 0:
  119. param = line[line.find("(") + 1:line.find(")")]
  120. print "INFO: ---- Found export", filename, param
  121. break
  122. else:
  123. if line != "int sr_kemi_modules_add(sr_kemi_t *klist)\n":
  124. print "ERR: Possible error at line: ", filename, line
  125. exit()
  126. return param
  127. def extract_c_file_kemi_export_lines(self, filename, lines, export_name):
  128. list_functions = []
  129. find_start = True
  130. for line in lines:
  131. if find_start and line.find("static sr_kemi_t " + export_name + "[]") >= 0:
  132. find_start = False
  133. elif not find_start:
  134. if line.find("};") >= 0:
  135. break
  136. line = line.lstrip(" \t")
  137. line = line.rstrip()
  138. list_functions.append(line)
  139. if len(list_functions) < 1:
  140. print "ERR: Couldn't parse file for exported functions: ", export_name
  141. exit()
  142. parsed_list = self.parse_kemi_export_c_lines(filename, list_functions)
  143. return parsed_list
  144. def parse_kemi_export_c_lines(self, filename, lines):
  145. elements = []
  146. function_lines = []
  147. temp_function = ""
  148. # We look for str_init which would be the start of each exported function
  149. for line in lines:
  150. if line.find("str_init") >= 0:
  151. if temp_function != "":
  152. function_lines.append(temp_function)
  153. temp_function = ""
  154. temp_function += line
  155. if temp_function != "":
  156. function_lines.append(temp_function)
  157. # Now we parse each exported function to extract its declaration
  158. for func in function_lines:
  159. function_lines_split = func.split(",{")
  160. if len(function_lines_split) < 2:
  161. print "ERR: Incorrect function line", func
  162. exit()
  163. declarations = function_lines_split[0].split(",")
  164. params = function_lines_split[1]
  165. # Extract the content from the definitions
  166. val_module = declarations[0]
  167. val_module = val_module[val_module.find('("') + 2:val_module.find('")')]
  168. val_function = declarations[1]
  169. val_function = val_function[val_function.find('("') + 2:val_function.find('")')]
  170. if declarations[2] == "SR_KEMIP_INT":
  171. val_return = "int"
  172. elif declarations[2] == "SR_KEMIP_STR":
  173. val_return = "string"
  174. elif declarations[2] == "SR_KEMIP_NONE":
  175. val_return = "void"
  176. elif declarations[2] == "SR_KEMIP_BOOL":
  177. val_return = "bool"
  178. else:
  179. print "ERR: Invalid return value", declarations[2], func
  180. exit()
  181. val_c_function = declarations[3].strip()
  182. # Count how many parameters the KEMI C function expects
  183. val_params = []
  184. itr = 0
  185. for val in params.rstrip("},").split(","):
  186. itr += 1
  187. # KEMI function has a maximum of 6 params
  188. if itr > 6:
  189. break
  190. pm = val.strip()
  191. if pm == "SR_KEMIP_INT":
  192. val_params.append("int")
  193. elif pm == "SR_KEMIP_STR":
  194. val_params.append("str")
  195. elif pm == "SR_KEMIP_NONE":
  196. continue
  197. else:
  198. print "Invalid return value", declarations[2], func
  199. exit()
  200. if itr != 6:
  201. print "ERR: Couldn't iterate the params: ", params
  202. exit()
  203. param_string = self.find_c_function_params(filename, val_c_function, val_return)
  204. param_string = self.prettify_params_list(val_function, param_string, val_params)
  205. elements.append({"module": val_module, "name": val_function, "ret": val_return, "params": param_string})
  206. return elements
  207. def prettify_params_list(self, function_name, function_declaration, kemi_types):
  208. # Validate the quantity and types of declaration vs export
  209. if function_declaration == "" and len(kemi_types) == 0:
  210. return ""
  211. params = function_declaration.split(",")
  212. if params[0].find("sip_msg_t") >= 0 or params[0].find("struct sip_msg") >= 0:
  213. params.pop(0)
  214. if len(params) != len(kemi_types):
  215. print "ERR: Mismatching quantity of params. Declaration for", function_name, ":", function_declaration, "KEMI:", kemi_types
  216. exit()
  217. for declared, type in zip(params, kemi_types):
  218. declared = declared.replace("*", "")
  219. declared = declared.strip().split(" ")[0]
  220. if declared != type:
  221. print "ERR: Mismatching type of params for", function_name, ":", function_declaration, " | ", kemi_types, " | Declared: ", declared, " Type: ", type
  222. exit()
  223. param_string = ""
  224. for param in params:
  225. param = param.strip()
  226. param = param.replace("*", "")
  227. if param[:3] == "str":
  228. temp = param.split(" ")
  229. param = "str" + ' "' + temp[1] + '"'
  230. param_string += param + ", "
  231. # Clean up the presentation of the params
  232. param_string = param_string.rstrip(", ")
  233. return param_string
  234. def find_c_function_params(self, filename, function_name, return_type):
  235. # First we try with the same file to find the declaration
  236. param_string = self.search_file_for_function_declaration(filename, function_name, return_type)
  237. # If we couldn't find it, let's try all files in the same folder as the first file
  238. if param_string:
  239. return param_string
  240. else:
  241. files = self.list_c_files_in_directory(os.path.dirname(filename))
  242. for file in files:
  243. param_string = self.search_file_for_function_declaration(file, function_name, return_type)
  244. if param_string:
  245. return param_string
  246. if function_name in self.macro_functions:
  247. return self.macro_functions[function_name]
  248. print "ERR: Couldn't find the function declaration", filename, function_name, return_type
  249. exit()
  250. def search_file_for_function_declaration(self, filename, function_name, return_type):
  251. # print "Searching file", filename, "for", function_name
  252. with open(filename) as f:
  253. lines = f.readlines()
  254. param_string = None
  255. found = False
  256. temp_string = ""
  257. # KEMI has some magic where most functions actually return INTs but KEMI maps them to void/bool
  258. if return_type == "void" or return_type == "bool":
  259. return_type = "int"
  260. # Look for declarations in format: static? return_type function_name(
  261. r = re.compile("^(?:static )?" + return_type + "[ \t]*(" + function_name + ")\(")
  262. for line in lines:
  263. m = r.match(line)
  264. if m:
  265. found = True
  266. if found:
  267. temp_string += line
  268. if line.find("{") >= 0:
  269. param_string = temp_string[temp_string.find('(') + 1:temp_string.find(')')]
  270. break
  271. return param_string
  272. def list_c_files_in_directory(self, path):
  273. matches = []
  274. for root, dirnames, filenames in os.walk(path):
  275. for filename in fnmatch.filter(filenames, '*.c'):
  276. matches.append(os.path.join(root, filename))
  277. return matches
  278. if __name__ == "__main__":
  279. reload(sys)
  280. sys.setdefaultencoding('utf8')
  281. try:
  282. if not os.path.isdir(sys.argv[1]):
  283. raise Exception('Not a valid directory')
  284. except:
  285. print "Please provide the path to the Kamailio src folder as the first argument"
  286. exit()
  287. print "Parsing the source"
  288. parser = KemiFileExportParser()
  289. data = parser.generate_kemi_export_list(sys.argv[1].rstrip("/"))
  290. fgen = ModuleDocGenerator()
  291. fgen.execute(data)
  292. print "Done"