gdscript_tokenizer.cpp 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532
  1. /*************************************************************************/
  2. /* gdscript_tokenizer.cpp */
  3. /*************************************************************************/
  4. /* This file is part of: */
  5. /* GODOT ENGINE */
  6. /* https://godotengine.org */
  7. /*************************************************************************/
  8. /* Copyright (c) 2007-2019 Juan Linietsky, Ariel Manzur. */
  9. /* Copyright (c) 2014-2019 Godot Engine contributors (cf. AUTHORS.md) */
  10. /* */
  11. /* Permission is hereby granted, free of charge, to any person obtaining */
  12. /* a copy of this software and associated documentation files (the */
  13. /* "Software"), to deal in the Software without restriction, including */
  14. /* without limitation the rights to use, copy, modify, merge, publish, */
  15. /* distribute, sublicense, and/or sell copies of the Software, and to */
  16. /* permit persons to whom the Software is furnished to do so, subject to */
  17. /* the following conditions: */
  18. /* */
  19. /* The above copyright notice and this permission notice shall be */
  20. /* included in all copies or substantial portions of the Software. */
  21. /* */
  22. /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
  23. /* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
  24. /* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
  25. /* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
  26. /* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
  27. /* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
  28. /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
  29. /*************************************************************************/
  30. #include "gdscript_tokenizer.h"
  31. #include "core/io/marshalls.h"
  32. #include "core/map.h"
  33. #include "core/print_string.h"
  34. #include "gdscript_functions.h"
  35. const char *GDScriptTokenizer::token_names[TK_MAX] = {
  36. "Empty",
  37. "Identifier",
  38. "Constant",
  39. "Self",
  40. "Built-In Type",
  41. "Built-In Func",
  42. "In",
  43. "'=='",
  44. "'!='",
  45. "'<'",
  46. "'<='",
  47. "'>'",
  48. "'>='",
  49. "'and'",
  50. "'or'",
  51. "'not'",
  52. "'+'",
  53. "'-'",
  54. "'*'",
  55. "'/'",
  56. "'%'",
  57. "'<<'",
  58. "'>>'",
  59. "'='",
  60. "'+='",
  61. "'-='",
  62. "'*='",
  63. "'/='",
  64. "'%='",
  65. "'<<='",
  66. "'>>='",
  67. "'&='",
  68. "'|='",
  69. "'^='",
  70. "'&'",
  71. "'|'",
  72. "'^'",
  73. "'~'",
  74. //"Plus Plus",
  75. //"Minus Minus",
  76. "if",
  77. "elif",
  78. "else",
  79. "for",
  80. "while",
  81. "break",
  82. "continue",
  83. "pass",
  84. "return",
  85. "match",
  86. "func",
  87. "class",
  88. "class_name",
  89. "extends",
  90. "is",
  91. "onready",
  92. "tool",
  93. "static",
  94. "export",
  95. "setget",
  96. "const",
  97. "var",
  98. "as",
  99. "void",
  100. "enum",
  101. "preload",
  102. "assert",
  103. "yield",
  104. "signal",
  105. "breakpoint",
  106. "rpc",
  107. "sync",
  108. "master",
  109. "puppet",
  110. "slave",
  111. "remotesync",
  112. "mastersync",
  113. "puppetsync",
  114. "'['",
  115. "']'",
  116. "'{'",
  117. "'}'",
  118. "'('",
  119. "')'",
  120. "','",
  121. "';'",
  122. "'.'",
  123. "'?'",
  124. "':'",
  125. "'$'",
  126. "'->'",
  127. "'\\n'",
  128. "PI",
  129. "TAU",
  130. "_",
  131. "INF",
  132. "NAN",
  133. "Error",
  134. "EOF",
  135. "Cursor"
  136. };
  137. struct _bit {
  138. Variant::Type type;
  139. const char *text;
  140. };
  141. //built in types
  142. static const _bit _type_list[] = {
  143. //types
  144. { Variant::BOOL, "bool" },
  145. { Variant::INT, "int" },
  146. { Variant::REAL, "float" },
  147. { Variant::STRING, "String" },
  148. { Variant::VECTOR2, "Vector2" },
  149. { Variant::RECT2, "Rect2" },
  150. { Variant::TRANSFORM2D, "Transform2D" },
  151. { Variant::VECTOR3, "Vector3" },
  152. { Variant::AABB, "AABB" },
  153. { Variant::PLANE, "Plane" },
  154. { Variant::QUAT, "Quat" },
  155. { Variant::BASIS, "Basis" },
  156. { Variant::TRANSFORM, "Transform" },
  157. { Variant::COLOR, "Color" },
  158. { Variant::_RID, "RID" },
  159. { Variant::OBJECT, "Object" },
  160. { Variant::NODE_PATH, "NodePath" },
  161. { Variant::DICTIONARY, "Dictionary" },
  162. { Variant::ARRAY, "Array" },
  163. { Variant::POOL_BYTE_ARRAY, "PoolByteArray" },
  164. { Variant::POOL_INT_ARRAY, "PoolIntArray" },
  165. { Variant::POOL_REAL_ARRAY, "PoolRealArray" },
  166. { Variant::POOL_STRING_ARRAY, "PoolStringArray" },
  167. { Variant::POOL_VECTOR2_ARRAY, "PoolVector2Array" },
  168. { Variant::POOL_VECTOR3_ARRAY, "PoolVector3Array" },
  169. { Variant::POOL_COLOR_ARRAY, "PoolColorArray" },
  170. { Variant::VARIANT_MAX, NULL },
  171. };
  172. struct _kws {
  173. GDScriptTokenizer::Token token;
  174. const char *text;
  175. };
  176. static const _kws _keyword_list[] = {
  177. //ops
  178. { GDScriptTokenizer::TK_OP_IN, "in" },
  179. { GDScriptTokenizer::TK_OP_NOT, "not" },
  180. { GDScriptTokenizer::TK_OP_OR, "or" },
  181. { GDScriptTokenizer::TK_OP_AND, "and" },
  182. //func
  183. { GDScriptTokenizer::TK_PR_FUNCTION, "func" },
  184. { GDScriptTokenizer::TK_PR_CLASS, "class" },
  185. { GDScriptTokenizer::TK_PR_CLASS_NAME, "class_name" },
  186. { GDScriptTokenizer::TK_PR_EXTENDS, "extends" },
  187. { GDScriptTokenizer::TK_PR_IS, "is" },
  188. { GDScriptTokenizer::TK_PR_ONREADY, "onready" },
  189. { GDScriptTokenizer::TK_PR_TOOL, "tool" },
  190. { GDScriptTokenizer::TK_PR_STATIC, "static" },
  191. { GDScriptTokenizer::TK_PR_EXPORT, "export" },
  192. { GDScriptTokenizer::TK_PR_SETGET, "setget" },
  193. { GDScriptTokenizer::TK_PR_VAR, "var" },
  194. { GDScriptTokenizer::TK_PR_AS, "as" },
  195. { GDScriptTokenizer::TK_PR_VOID, "void" },
  196. { GDScriptTokenizer::TK_PR_PRELOAD, "preload" },
  197. { GDScriptTokenizer::TK_PR_ASSERT, "assert" },
  198. { GDScriptTokenizer::TK_PR_YIELD, "yield" },
  199. { GDScriptTokenizer::TK_PR_SIGNAL, "signal" },
  200. { GDScriptTokenizer::TK_PR_BREAKPOINT, "breakpoint" },
  201. { GDScriptTokenizer::TK_PR_REMOTE, "remote" },
  202. { GDScriptTokenizer::TK_PR_MASTER, "master" },
  203. { GDScriptTokenizer::TK_PR_SLAVE, "slave" },
  204. { GDScriptTokenizer::TK_PR_PUPPET, "puppet" },
  205. { GDScriptTokenizer::TK_PR_SYNC, "sync" },
  206. { GDScriptTokenizer::TK_PR_REMOTESYNC, "remotesync" },
  207. { GDScriptTokenizer::TK_PR_MASTERSYNC, "mastersync" },
  208. { GDScriptTokenizer::TK_PR_PUPPETSYNC, "puppetsync" },
  209. { GDScriptTokenizer::TK_PR_CONST, "const" },
  210. { GDScriptTokenizer::TK_PR_ENUM, "enum" },
  211. //controlflow
  212. { GDScriptTokenizer::TK_CF_IF, "if" },
  213. { GDScriptTokenizer::TK_CF_ELIF, "elif" },
  214. { GDScriptTokenizer::TK_CF_ELSE, "else" },
  215. { GDScriptTokenizer::TK_CF_FOR, "for" },
  216. { GDScriptTokenizer::TK_CF_WHILE, "while" },
  217. { GDScriptTokenizer::TK_CF_BREAK, "break" },
  218. { GDScriptTokenizer::TK_CF_CONTINUE, "continue" },
  219. { GDScriptTokenizer::TK_CF_RETURN, "return" },
  220. { GDScriptTokenizer::TK_CF_MATCH, "match" },
  221. { GDScriptTokenizer::TK_CF_PASS, "pass" },
  222. { GDScriptTokenizer::TK_SELF, "self" },
  223. { GDScriptTokenizer::TK_CONST_PI, "PI" },
  224. { GDScriptTokenizer::TK_CONST_TAU, "TAU" },
  225. { GDScriptTokenizer::TK_WILDCARD, "_" },
  226. { GDScriptTokenizer::TK_CONST_INF, "INF" },
  227. { GDScriptTokenizer::TK_CONST_NAN, "NAN" },
  228. { GDScriptTokenizer::TK_ERROR, NULL }
  229. };
  230. const char *GDScriptTokenizer::get_token_name(Token p_token) {
  231. ERR_FAIL_INDEX_V(p_token, TK_MAX, "<error>");
  232. return token_names[p_token];
  233. }
  234. bool GDScriptTokenizer::is_token_literal(int p_offset, bool variable_safe) const {
  235. switch (get_token(p_offset)) {
  236. // Can always be literal:
  237. case TK_IDENTIFIER:
  238. case TK_PR_ONREADY:
  239. case TK_PR_TOOL:
  240. case TK_PR_STATIC:
  241. case TK_PR_EXPORT:
  242. case TK_PR_SETGET:
  243. case TK_PR_SIGNAL:
  244. case TK_PR_REMOTE:
  245. case TK_PR_MASTER:
  246. case TK_PR_PUPPET:
  247. case TK_PR_SYNC:
  248. case TK_PR_REMOTESYNC:
  249. case TK_PR_MASTERSYNC:
  250. case TK_PR_PUPPETSYNC:
  251. return true;
  252. // Literal for non-variables only:
  253. case TK_BUILT_IN_TYPE:
  254. case TK_BUILT_IN_FUNC:
  255. case TK_OP_IN:
  256. //case TK_OP_NOT:
  257. //case TK_OP_OR:
  258. //case TK_OP_AND:
  259. case TK_PR_CLASS:
  260. case TK_PR_CONST:
  261. case TK_PR_ENUM:
  262. case TK_PR_PRELOAD:
  263. case TK_PR_FUNCTION:
  264. case TK_PR_EXTENDS:
  265. case TK_PR_ASSERT:
  266. case TK_PR_YIELD:
  267. case TK_PR_VAR:
  268. case TK_CF_IF:
  269. case TK_CF_ELIF:
  270. case TK_CF_ELSE:
  271. case TK_CF_FOR:
  272. case TK_CF_WHILE:
  273. case TK_CF_BREAK:
  274. case TK_CF_CONTINUE:
  275. case TK_CF_RETURN:
  276. case TK_CF_MATCH:
  277. case TK_CF_PASS:
  278. case TK_SELF:
  279. case TK_CONST_PI:
  280. case TK_CONST_TAU:
  281. case TK_WILDCARD:
  282. case TK_CONST_INF:
  283. case TK_CONST_NAN:
  284. case TK_ERROR:
  285. return !variable_safe;
  286. case TK_CONSTANT: {
  287. switch (get_token_constant(p_offset).get_type()) {
  288. case Variant::NIL:
  289. case Variant::BOOL:
  290. return true;
  291. default:
  292. return false;
  293. }
  294. }
  295. default:
  296. return false;
  297. }
  298. }
  299. StringName GDScriptTokenizer::get_token_literal(int p_offset) const {
  300. Token token = get_token(p_offset);
  301. switch (token) {
  302. case TK_IDENTIFIER:
  303. return get_token_identifier(p_offset);
  304. case TK_BUILT_IN_TYPE: {
  305. Variant::Type type = get_token_type(p_offset);
  306. int idx = 0;
  307. while (_type_list[idx].text) {
  308. if (type == _type_list[idx].type) {
  309. return _type_list[idx].text;
  310. }
  311. idx++;
  312. }
  313. } break; // Shouldn't get here, stuff happens
  314. case TK_BUILT_IN_FUNC:
  315. return GDScriptFunctions::get_func_name(get_token_built_in_func(p_offset));
  316. case TK_CONSTANT: {
  317. const Variant value = get_token_constant(p_offset);
  318. switch (value.get_type()) {
  319. case Variant::NIL:
  320. return "null";
  321. case Variant::BOOL:
  322. return value ? "true" : "false";
  323. default: {
  324. }
  325. }
  326. }
  327. case TK_OP_AND:
  328. case TK_OP_OR:
  329. break; // Don't get into default, since they can be non-literal
  330. default: {
  331. int idx = 0;
  332. while (_keyword_list[idx].text) {
  333. if (token == _keyword_list[idx].token) {
  334. return _keyword_list[idx].text;
  335. }
  336. idx++;
  337. }
  338. }
  339. }
  340. ERR_FAIL_V_MSG("", "Failed to get token literal.");
  341. }
  342. static bool _is_text_char(CharType c) {
  343. return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
  344. }
  345. static bool _is_number(CharType c) {
  346. return (c >= '0' && c <= '9');
  347. }
  348. static bool _is_hex(CharType c) {
  349. return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
  350. }
  351. static bool _is_bin(CharType c) {
  352. return (c == '0' || c == '1');
  353. }
  354. void GDScriptTokenizerText::_make_token(Token p_type) {
  355. TokenData &tk = tk_rb[tk_rb_pos];
  356. tk.type = p_type;
  357. tk.line = line;
  358. tk.col = column;
  359. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  360. }
  361. void GDScriptTokenizerText::_make_identifier(const StringName &p_identifier) {
  362. TokenData &tk = tk_rb[tk_rb_pos];
  363. tk.type = TK_IDENTIFIER;
  364. tk.identifier = p_identifier;
  365. tk.line = line;
  366. tk.col = column;
  367. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  368. }
  369. void GDScriptTokenizerText::_make_built_in_func(GDScriptFunctions::Function p_func) {
  370. TokenData &tk = tk_rb[tk_rb_pos];
  371. tk.type = TK_BUILT_IN_FUNC;
  372. tk.func = p_func;
  373. tk.line = line;
  374. tk.col = column;
  375. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  376. }
  377. void GDScriptTokenizerText::_make_constant(const Variant &p_constant) {
  378. TokenData &tk = tk_rb[tk_rb_pos];
  379. tk.type = TK_CONSTANT;
  380. tk.constant = p_constant;
  381. tk.line = line;
  382. tk.col = column;
  383. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  384. }
  385. void GDScriptTokenizerText::_make_type(const Variant::Type &p_type) {
  386. TokenData &tk = tk_rb[tk_rb_pos];
  387. tk.type = TK_BUILT_IN_TYPE;
  388. tk.vtype = p_type;
  389. tk.line = line;
  390. tk.col = column;
  391. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  392. }
  393. void GDScriptTokenizerText::_make_error(const String &p_error) {
  394. error_flag = true;
  395. last_error = p_error;
  396. TokenData &tk = tk_rb[tk_rb_pos];
  397. tk.type = TK_ERROR;
  398. tk.constant = p_error;
  399. tk.line = line;
  400. tk.col = column;
  401. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  402. }
  403. void GDScriptTokenizerText::_make_newline(int p_indentation, int p_tabs) {
  404. TokenData &tk = tk_rb[tk_rb_pos];
  405. tk.type = TK_NEWLINE;
  406. tk.constant = Vector2(p_indentation, p_tabs);
  407. tk.line = line;
  408. tk.col = column;
  409. tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
  410. }
  411. void GDScriptTokenizerText::_advance() {
  412. if (error_flag) {
  413. //parser broke
  414. _make_error(last_error);
  415. return;
  416. }
  417. if (code_pos >= len) {
  418. _make_token(TK_EOF);
  419. return;
  420. }
  421. #define GETCHAR(m_ofs) ((m_ofs + code_pos) >= len ? 0 : _code[m_ofs + code_pos])
  422. #define INCPOS(m_amount) \
  423. { \
  424. code_pos += m_amount; \
  425. column += m_amount; \
  426. }
  427. while (true) {
  428. bool is_node_path = false;
  429. StringMode string_mode = STRING_DOUBLE_QUOTE;
  430. switch (GETCHAR(0)) {
  431. case 0:
  432. _make_token(TK_EOF);
  433. break;
  434. case '\\':
  435. INCPOS(1);
  436. if (GETCHAR(0) == '\r') {
  437. INCPOS(1);
  438. }
  439. if (GETCHAR(0) != '\n') {
  440. _make_error("Expected newline after '\\'.");
  441. return;
  442. }
  443. INCPOS(1);
  444. line++;
  445. while (GETCHAR(0) == ' ' || GETCHAR(0) == '\t') {
  446. INCPOS(1);
  447. }
  448. continue;
  449. case '\t':
  450. case '\r':
  451. case ' ':
  452. INCPOS(1);
  453. continue;
  454. case '#': { // line comment skip
  455. #ifdef DEBUG_ENABLED
  456. String comment;
  457. #endif // DEBUG_ENABLED
  458. while (GETCHAR(0) != '\n') {
  459. #ifdef DEBUG_ENABLED
  460. comment += GETCHAR(0);
  461. #endif // DEBUG_ENABLED
  462. code_pos++;
  463. if (GETCHAR(0) == 0) { //end of file
  464. //_make_error("Unterminated Comment");
  465. _make_token(TK_EOF);
  466. return;
  467. }
  468. }
  469. #ifdef DEBUG_ENABLED
  470. String comment_content = comment.trim_prefix("#").trim_prefix(" ");
  471. if (comment_content.begins_with("warning-ignore:")) {
  472. String code = comment_content.get_slice(":", 1);
  473. warning_skips.push_back(Pair<int, String>(line, code.strip_edges().to_lower()));
  474. } else if (comment_content.begins_with("warning-ignore-all:")) {
  475. String code = comment_content.get_slice(":", 1);
  476. warning_global_skips.insert(code.strip_edges().to_lower());
  477. } else if (comment_content.strip_edges() == "warnings-disable") {
  478. ignore_warnings = true;
  479. }
  480. #endif // DEBUG_ENABLED
  481. FALLTHROUGH;
  482. }
  483. case '\n': {
  484. line++;
  485. INCPOS(1);
  486. bool used_spaces = false;
  487. int tabs = 0;
  488. column = 1;
  489. int i = 0;
  490. while (true) {
  491. if (GETCHAR(i) == ' ') {
  492. i++;
  493. used_spaces = true;
  494. } else if (GETCHAR(i) == '\t') {
  495. if (used_spaces) {
  496. _make_error("Spaces used before tabs on a line");
  497. return;
  498. }
  499. i++;
  500. tabs++;
  501. } else {
  502. break; // not indentation anymore
  503. }
  504. }
  505. _make_newline(i, tabs);
  506. return;
  507. }
  508. case '/': {
  509. switch (GETCHAR(1)) {
  510. case '=': { // diveq
  511. _make_token(TK_OP_ASSIGN_DIV);
  512. INCPOS(1);
  513. } break;
  514. default:
  515. _make_token(TK_OP_DIV);
  516. }
  517. } break;
  518. case '=': {
  519. if (GETCHAR(1) == '=') {
  520. _make_token(TK_OP_EQUAL);
  521. INCPOS(1);
  522. } else
  523. _make_token(TK_OP_ASSIGN);
  524. } break;
  525. case '<': {
  526. if (GETCHAR(1) == '=') {
  527. _make_token(TK_OP_LESS_EQUAL);
  528. INCPOS(1);
  529. } else if (GETCHAR(1) == '<') {
  530. if (GETCHAR(2) == '=') {
  531. _make_token(TK_OP_ASSIGN_SHIFT_LEFT);
  532. INCPOS(1);
  533. } else {
  534. _make_token(TK_OP_SHIFT_LEFT);
  535. }
  536. INCPOS(1);
  537. } else
  538. _make_token(TK_OP_LESS);
  539. } break;
  540. case '>': {
  541. if (GETCHAR(1) == '=') {
  542. _make_token(TK_OP_GREATER_EQUAL);
  543. INCPOS(1);
  544. } else if (GETCHAR(1) == '>') {
  545. if (GETCHAR(2) == '=') {
  546. _make_token(TK_OP_ASSIGN_SHIFT_RIGHT);
  547. INCPOS(1);
  548. } else {
  549. _make_token(TK_OP_SHIFT_RIGHT);
  550. }
  551. INCPOS(1);
  552. } else {
  553. _make_token(TK_OP_GREATER);
  554. }
  555. } break;
  556. case '!': {
  557. if (GETCHAR(1) == '=') {
  558. _make_token(TK_OP_NOT_EQUAL);
  559. INCPOS(1);
  560. } else {
  561. _make_token(TK_OP_NOT);
  562. }
  563. } break;
  564. //case '"' //string - no strings in shader
  565. //case '\'' //string - no strings in shader
  566. case '{':
  567. _make_token(TK_CURLY_BRACKET_OPEN);
  568. break;
  569. case '}':
  570. _make_token(TK_CURLY_BRACKET_CLOSE);
  571. break;
  572. case '[':
  573. _make_token(TK_BRACKET_OPEN);
  574. break;
  575. case ']':
  576. _make_token(TK_BRACKET_CLOSE);
  577. break;
  578. case '(':
  579. _make_token(TK_PARENTHESIS_OPEN);
  580. break;
  581. case ')':
  582. _make_token(TK_PARENTHESIS_CLOSE);
  583. break;
  584. case ',':
  585. _make_token(TK_COMMA);
  586. break;
  587. case ';':
  588. _make_token(TK_SEMICOLON);
  589. break;
  590. case '?':
  591. _make_token(TK_QUESTION_MARK);
  592. break;
  593. case ':':
  594. _make_token(TK_COLON); //for methods maybe but now useless.
  595. break;
  596. case '$':
  597. _make_token(TK_DOLLAR); //for the get_node() shortener
  598. break;
  599. case '^': {
  600. if (GETCHAR(1) == '=') {
  601. _make_token(TK_OP_ASSIGN_BIT_XOR);
  602. INCPOS(1);
  603. } else {
  604. _make_token(TK_OP_BIT_XOR);
  605. }
  606. } break;
  607. case '~':
  608. _make_token(TK_OP_BIT_INVERT);
  609. break;
  610. case '&': {
  611. if (GETCHAR(1) == '&') {
  612. _make_token(TK_OP_AND);
  613. INCPOS(1);
  614. } else if (GETCHAR(1) == '=') {
  615. _make_token(TK_OP_ASSIGN_BIT_AND);
  616. INCPOS(1);
  617. } else {
  618. _make_token(TK_OP_BIT_AND);
  619. }
  620. } break;
  621. case '|': {
  622. if (GETCHAR(1) == '|') {
  623. _make_token(TK_OP_OR);
  624. INCPOS(1);
  625. } else if (GETCHAR(1) == '=') {
  626. _make_token(TK_OP_ASSIGN_BIT_OR);
  627. INCPOS(1);
  628. } else {
  629. _make_token(TK_OP_BIT_OR);
  630. }
  631. } break;
  632. case '*': {
  633. if (GETCHAR(1) == '=') {
  634. _make_token(TK_OP_ASSIGN_MUL);
  635. INCPOS(1);
  636. } else {
  637. _make_token(TK_OP_MUL);
  638. }
  639. } break;
  640. case '+': {
  641. if (GETCHAR(1) == '=') {
  642. _make_token(TK_OP_ASSIGN_ADD);
  643. INCPOS(1);
  644. /*
  645. } else if (GETCHAR(1)=='+') {
  646. _make_token(TK_OP_PLUS_PLUS);
  647. INCPOS(1);
  648. */
  649. } else {
  650. _make_token(TK_OP_ADD);
  651. }
  652. } break;
  653. case '-': {
  654. if (GETCHAR(1) == '=') {
  655. _make_token(TK_OP_ASSIGN_SUB);
  656. INCPOS(1);
  657. } else if (GETCHAR(1) == '>') {
  658. _make_token(TK_FORWARD_ARROW);
  659. INCPOS(1);
  660. } else {
  661. _make_token(TK_OP_SUB);
  662. }
  663. } break;
  664. case '%': {
  665. if (GETCHAR(1) == '=') {
  666. _make_token(TK_OP_ASSIGN_MOD);
  667. INCPOS(1);
  668. } else {
  669. _make_token(TK_OP_MOD);
  670. }
  671. } break;
  672. case '@':
  673. if (CharType(GETCHAR(1)) != '"' && CharType(GETCHAR(1)) != '\'') {
  674. _make_error("Unexpected '@'");
  675. return;
  676. }
  677. INCPOS(1);
  678. is_node_path = true;
  679. FALLTHROUGH;
  680. case '\'':
  681. case '"': {
  682. if (GETCHAR(0) == '\'')
  683. string_mode = STRING_SINGLE_QUOTE;
  684. int i = 1;
  685. if (string_mode == STRING_DOUBLE_QUOTE && GETCHAR(i) == '"' && GETCHAR(i + 1) == '"') {
  686. i += 2;
  687. string_mode = STRING_MULTILINE;
  688. }
  689. String str;
  690. while (true) {
  691. if (CharType(GETCHAR(i)) == 0) {
  692. _make_error("Unterminated String");
  693. return;
  694. } else if (string_mode == STRING_DOUBLE_QUOTE && CharType(GETCHAR(i)) == '"') {
  695. break;
  696. } else if (string_mode == STRING_SINGLE_QUOTE && CharType(GETCHAR(i)) == '\'') {
  697. break;
  698. } else if (string_mode == STRING_MULTILINE && CharType(GETCHAR(i)) == '\"' && CharType(GETCHAR(i + 1)) == '\"' && CharType(GETCHAR(i + 2)) == '\"') {
  699. i += 2;
  700. break;
  701. } else if (string_mode != STRING_MULTILINE && CharType(GETCHAR(i)) == '\n') {
  702. _make_error("Unexpected EOL at String.");
  703. return;
  704. } else if (CharType(GETCHAR(i)) == 0xFFFF) {
  705. //string ends here, next will be TK
  706. i--;
  707. break;
  708. } else if (CharType(GETCHAR(i)) == '\\') {
  709. //escaped characters...
  710. i++;
  711. CharType next = GETCHAR(i);
  712. if (next == 0) {
  713. _make_error("Unterminated String");
  714. return;
  715. }
  716. CharType res = 0;
  717. switch (next) {
  718. case 'a': res = 7; break;
  719. case 'b': res = 8; break;
  720. case 't': res = 9; break;
  721. case 'n': res = 10; break;
  722. case 'v': res = 11; break;
  723. case 'f': res = 12; break;
  724. case 'r': res = 13; break;
  725. case '\'': res = '\''; break;
  726. case '\"': res = '\"'; break;
  727. case '\\': res = '\\'; break;
  728. case '/':
  729. res = '/';
  730. break; //wtf
  731. case 'u': {
  732. //hexnumbarh - oct is deprecated
  733. i += 1;
  734. for (int j = 0; j < 4; j++) {
  735. CharType c = GETCHAR(i + j);
  736. if (c == 0) {
  737. _make_error("Unterminated String");
  738. return;
  739. }
  740. if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) {
  741. _make_error("Malformed hex constant in string");
  742. return;
  743. }
  744. CharType v;
  745. if (c >= '0' && c <= '9') {
  746. v = c - '0';
  747. } else if (c >= 'a' && c <= 'f') {
  748. v = c - 'a';
  749. v += 10;
  750. } else if (c >= 'A' && c <= 'F') {
  751. v = c - 'A';
  752. v += 10;
  753. } else {
  754. ERR_PRINT("BUG");
  755. v = 0;
  756. }
  757. res <<= 4;
  758. res |= v;
  759. }
  760. i += 3;
  761. } break;
  762. default: {
  763. _make_error("Invalid escape sequence");
  764. return;
  765. } break;
  766. }
  767. str += res;
  768. } else {
  769. if (CharType(GETCHAR(i)) == '\n') {
  770. line++;
  771. column = 1;
  772. }
  773. str += CharType(GETCHAR(i));
  774. }
  775. i++;
  776. }
  777. INCPOS(i);
  778. if (is_node_path) {
  779. _make_constant(NodePath(str));
  780. } else {
  781. _make_constant(str);
  782. }
  783. } break;
  784. case 0xFFFF: {
  785. _make_token(TK_CURSOR);
  786. } break;
  787. default: {
  788. if (_is_number(GETCHAR(0)) || (GETCHAR(0) == '.' && _is_number(GETCHAR(1)))) {
  789. // parse number
  790. bool period_found = false;
  791. bool exponent_found = false;
  792. bool hexa_found = false;
  793. bool bin_found = false;
  794. bool sign_found = false;
  795. String str;
  796. int i = 0;
  797. while (true) {
  798. if (GETCHAR(i) == '.') {
  799. if (period_found || exponent_found) {
  800. _make_error("Invalid numeric constant at '.'");
  801. return;
  802. } else if (bin_found) {
  803. _make_error("Invalid binary constant at '.'");
  804. return;
  805. } else if (hexa_found) {
  806. _make_error("Invalid hexadecimal constant at '.'");
  807. return;
  808. }
  809. period_found = true;
  810. } else if (GETCHAR(i) == 'x') {
  811. if (hexa_found || bin_found || str.length() != 1 || !((i == 1 && str[0] == '0') || (i == 2 && str[1] == '0' && str[0] == '-'))) {
  812. _make_error("Invalid numeric constant at 'x'");
  813. return;
  814. }
  815. hexa_found = true;
  816. } else if (GETCHAR(i) == 'b') {
  817. if (hexa_found || bin_found || str.length() != 1 || !((i == 1 && str[0] == '0') || (i == 2 && str[1] == '0' && str[0] == '-'))) {
  818. _make_error("Invalid numeric constant at 'b'");
  819. return;
  820. }
  821. bin_found = true;
  822. } else if (!hexa_found && GETCHAR(i) == 'e') {
  823. if (exponent_found || bin_found) {
  824. _make_error("Invalid numeric constant at 'e'");
  825. return;
  826. }
  827. exponent_found = true;
  828. } else if (_is_number(GETCHAR(i))) {
  829. //all ok
  830. } else if (hexa_found && _is_hex(GETCHAR(i))) {
  831. } else if (bin_found && _is_bin(GETCHAR(i))) {
  832. } else if ((GETCHAR(i) == '-' || GETCHAR(i) == '+') && exponent_found) {
  833. if (sign_found) {
  834. _make_error("Invalid numeric constant at '-'");
  835. return;
  836. }
  837. sign_found = true;
  838. } else if (GETCHAR(i) == '_') {
  839. i++;
  840. continue; // Included for readability, shouldn't be a part of the string
  841. } else
  842. break;
  843. str += CharType(GETCHAR(i));
  844. i++;
  845. }
  846. if (!(_is_number(str[str.length() - 1]) || (hexa_found && _is_hex(str[str.length() - 1])))) {
  847. _make_error("Invalid numeric constant: " + str);
  848. return;
  849. }
  850. INCPOS(i);
  851. if (hexa_found) {
  852. int64_t val = str.hex_to_int64();
  853. _make_constant(val);
  854. } else if (bin_found) {
  855. int64_t val = str.bin_to_int64();
  856. _make_constant(val);
  857. } else if (period_found || exponent_found) {
  858. double val = str.to_double();
  859. _make_constant(val);
  860. } else {
  861. int64_t val = str.to_int64();
  862. _make_constant(val);
  863. }
  864. return;
  865. }
  866. if (GETCHAR(0) == '.') {
  867. //parse period
  868. _make_token(TK_PERIOD);
  869. break;
  870. }
  871. if (_is_text_char(GETCHAR(0))) {
  872. // parse identifier
  873. String str;
  874. str += CharType(GETCHAR(0));
  875. int i = 1;
  876. while (_is_text_char(GETCHAR(i))) {
  877. str += CharType(GETCHAR(i));
  878. i++;
  879. }
  880. bool identifier = false;
  881. if (str == "null") {
  882. _make_constant(Variant());
  883. } else if (str == "true") {
  884. _make_constant(true);
  885. } else if (str == "false") {
  886. _make_constant(false);
  887. } else {
  888. bool found = false;
  889. {
  890. int idx = 0;
  891. while (_type_list[idx].text) {
  892. if (str == _type_list[idx].text) {
  893. _make_type(_type_list[idx].type);
  894. found = true;
  895. break;
  896. }
  897. idx++;
  898. }
  899. }
  900. if (!found) {
  901. //built in func?
  902. for (int j = 0; j < GDScriptFunctions::FUNC_MAX; j++) {
  903. if (str == GDScriptFunctions::get_func_name(GDScriptFunctions::Function(j))) {
  904. _make_built_in_func(GDScriptFunctions::Function(j));
  905. found = true;
  906. break;
  907. }
  908. }
  909. }
  910. if (!found) {
  911. //keyword
  912. int idx = 0;
  913. found = false;
  914. while (_keyword_list[idx].text) {
  915. if (str == _keyword_list[idx].text) {
  916. _make_token(_keyword_list[idx].token);
  917. found = true;
  918. break;
  919. }
  920. idx++;
  921. }
  922. }
  923. if (!found)
  924. identifier = true;
  925. }
  926. if (identifier) {
  927. _make_identifier(str);
  928. }
  929. INCPOS(str.length());
  930. return;
  931. }
  932. _make_error("Unknown character");
  933. return;
  934. } break;
  935. }
  936. INCPOS(1);
  937. break;
  938. }
  939. }
  940. void GDScriptTokenizerText::set_code(const String &p_code) {
  941. code = p_code;
  942. len = p_code.length();
  943. if (len) {
  944. _code = &code[0];
  945. } else {
  946. _code = NULL;
  947. }
  948. code_pos = 0;
  949. line = 1; //it is stand-ar-ized that lines begin in 1 in code..
  950. column = 1; //the same holds for columns
  951. tk_rb_pos = 0;
  952. error_flag = false;
  953. #ifdef DEBUG_ENABLED
  954. ignore_warnings = false;
  955. #endif // DEBUG_ENABLED
  956. last_error = "";
  957. for (int i = 0; i < MAX_LOOKAHEAD + 1; i++)
  958. _advance();
  959. }
  960. GDScriptTokenizerText::Token GDScriptTokenizerText::get_token(int p_offset) const {
  961. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, TK_ERROR);
  962. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, TK_ERROR);
  963. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  964. return tk_rb[ofs].type;
  965. }
  966. int GDScriptTokenizerText::get_token_line(int p_offset) const {
  967. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, -1);
  968. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, -1);
  969. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  970. return tk_rb[ofs].line;
  971. }
  972. int GDScriptTokenizerText::get_token_column(int p_offset) const {
  973. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, -1);
  974. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, -1);
  975. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  976. return tk_rb[ofs].col;
  977. }
  978. const Variant &GDScriptTokenizerText::get_token_constant(int p_offset) const {
  979. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, tk_rb[0].constant);
  980. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, tk_rb[0].constant);
  981. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  982. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_CONSTANT, tk_rb[0].constant);
  983. return tk_rb[ofs].constant;
  984. }
  985. StringName GDScriptTokenizerText::get_token_identifier(int p_offset) const {
  986. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, StringName());
  987. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, StringName());
  988. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  989. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_IDENTIFIER, StringName());
  990. return tk_rb[ofs].identifier;
  991. }
  992. GDScriptFunctions::Function GDScriptTokenizerText::get_token_built_in_func(int p_offset) const {
  993. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, GDScriptFunctions::FUNC_MAX);
  994. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, GDScriptFunctions::FUNC_MAX);
  995. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  996. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_BUILT_IN_FUNC, GDScriptFunctions::FUNC_MAX);
  997. return tk_rb[ofs].func;
  998. }
  999. Variant::Type GDScriptTokenizerText::get_token_type(int p_offset) const {
  1000. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, Variant::NIL);
  1001. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, Variant::NIL);
  1002. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  1003. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_BUILT_IN_TYPE, Variant::NIL);
  1004. return tk_rb[ofs].vtype;
  1005. }
  1006. int GDScriptTokenizerText::get_token_line_indent(int p_offset) const {
  1007. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, 0);
  1008. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, 0);
  1009. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  1010. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_NEWLINE, 0);
  1011. return tk_rb[ofs].constant.operator Vector2().x;
  1012. }
  1013. int GDScriptTokenizerText::get_token_line_tab_indent(int p_offset) const {
  1014. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, 0);
  1015. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, 0);
  1016. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  1017. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_NEWLINE, 0);
  1018. return tk_rb[ofs].constant.operator Vector2().y;
  1019. }
  1020. String GDScriptTokenizerText::get_token_error(int p_offset) const {
  1021. ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, String());
  1022. ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, String());
  1023. int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
  1024. ERR_FAIL_COND_V(tk_rb[ofs].type != TK_ERROR, String());
  1025. return tk_rb[ofs].constant;
  1026. }
  1027. void GDScriptTokenizerText::advance(int p_amount) {
  1028. ERR_FAIL_COND(p_amount <= 0);
  1029. for (int i = 0; i < p_amount; i++)
  1030. _advance();
  1031. }
  1032. //////////////////////////////////////////////////////////////////////////////////////////////////////
  1033. #define BYTECODE_VERSION 13
  1034. Error GDScriptTokenizerBuffer::set_code_buffer(const Vector<uint8_t> &p_buffer) {
  1035. const uint8_t *buf = p_buffer.ptr();
  1036. int total_len = p_buffer.size();
  1037. ERR_FAIL_COND_V(p_buffer.size() < 24 || p_buffer[0] != 'G' || p_buffer[1] != 'D' || p_buffer[2] != 'S' || p_buffer[3] != 'C', ERR_INVALID_DATA);
  1038. int version = decode_uint32(&buf[4]);
  1039. ERR_FAIL_COND_V_MSG(version > BYTECODE_VERSION, ERR_INVALID_DATA, "Bytecode is too recent! Please use a newer engine version.");
  1040. int identifier_count = decode_uint32(&buf[8]);
  1041. int constant_count = decode_uint32(&buf[12]);
  1042. int line_count = decode_uint32(&buf[16]);
  1043. int token_count = decode_uint32(&buf[20]);
  1044. const uint8_t *b = &buf[24];
  1045. total_len -= 24;
  1046. identifiers.resize(identifier_count);
  1047. for (int i = 0; i < identifier_count; i++) {
  1048. int len = decode_uint32(b);
  1049. ERR_FAIL_COND_V(len > total_len, ERR_INVALID_DATA);
  1050. b += 4;
  1051. Vector<uint8_t> cs;
  1052. cs.resize(len);
  1053. for (int j = 0; j < len; j++) {
  1054. cs.write[j] = b[j] ^ 0xb6;
  1055. }
  1056. cs.write[cs.size() - 1] = 0;
  1057. String s;
  1058. s.parse_utf8((const char *)cs.ptr());
  1059. b += len;
  1060. total_len -= len + 4;
  1061. identifiers.write[i] = s;
  1062. }
  1063. constants.resize(constant_count);
  1064. for (int i = 0; i < constant_count; i++) {
  1065. Variant v;
  1066. int len;
  1067. // An object cannot be constant, never decode objects
  1068. Error err = decode_variant(v, b, total_len, &len, false);
  1069. if (err)
  1070. return err;
  1071. b += len;
  1072. total_len -= len;
  1073. constants.write[i] = v;
  1074. }
  1075. ERR_FAIL_COND_V(line_count * 8 > total_len, ERR_INVALID_DATA);
  1076. for (int i = 0; i < line_count; i++) {
  1077. uint32_t token = decode_uint32(b);
  1078. b += 4;
  1079. uint32_t linecol = decode_uint32(b);
  1080. b += 4;
  1081. lines.insert(token, linecol);
  1082. total_len -= 8;
  1083. }
  1084. tokens.resize(token_count);
  1085. for (int i = 0; i < token_count; i++) {
  1086. ERR_FAIL_COND_V(total_len < 1, ERR_INVALID_DATA);
  1087. if ((*b) & TOKEN_BYTE_MASK) { //little endian always
  1088. ERR_FAIL_COND_V(total_len < 4, ERR_INVALID_DATA);
  1089. tokens.write[i] = decode_uint32(b) & ~TOKEN_BYTE_MASK;
  1090. b += 4;
  1091. } else {
  1092. tokens.write[i] = *b;
  1093. b += 1;
  1094. total_len--;
  1095. }
  1096. }
  1097. token = 0;
  1098. return OK;
  1099. }
  1100. Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code) {
  1101. Vector<uint8_t> buf;
  1102. Map<StringName, int> identifier_map;
  1103. HashMap<Variant, int, VariantHasher, VariantComparator> constant_map;
  1104. Map<uint32_t, int> line_map;
  1105. Vector<uint32_t> token_array;
  1106. GDScriptTokenizerText tt;
  1107. tt.set_code(p_code);
  1108. int line = -1;
  1109. while (true) {
  1110. if (tt.get_token_line() != line) {
  1111. line = tt.get_token_line();
  1112. line_map[line] = token_array.size();
  1113. }
  1114. uint32_t token = tt.get_token();
  1115. switch (tt.get_token()) {
  1116. case TK_IDENTIFIER: {
  1117. StringName id = tt.get_token_identifier();
  1118. if (!identifier_map.has(id)) {
  1119. int idx = identifier_map.size();
  1120. identifier_map[id] = idx;
  1121. }
  1122. token |= identifier_map[id] << TOKEN_BITS;
  1123. } break;
  1124. case TK_CONSTANT: {
  1125. const Variant &c = tt.get_token_constant();
  1126. if (!constant_map.has(c)) {
  1127. int idx = constant_map.size();
  1128. constant_map[c] = idx;
  1129. }
  1130. token |= constant_map[c] << TOKEN_BITS;
  1131. } break;
  1132. case TK_BUILT_IN_TYPE: {
  1133. token |= tt.get_token_type() << TOKEN_BITS;
  1134. } break;
  1135. case TK_BUILT_IN_FUNC: {
  1136. token |= tt.get_token_built_in_func() << TOKEN_BITS;
  1137. } break;
  1138. case TK_NEWLINE: {
  1139. token |= tt.get_token_line_indent() << TOKEN_BITS;
  1140. } break;
  1141. case TK_ERROR: {
  1142. ERR_FAIL_V(Vector<uint8_t>());
  1143. } break;
  1144. default: {
  1145. }
  1146. };
  1147. token_array.push_back(token);
  1148. if (tt.get_token() == TK_EOF)
  1149. break;
  1150. tt.advance();
  1151. }
  1152. //reverse maps
  1153. Map<int, StringName> rev_identifier_map;
  1154. for (Map<StringName, int>::Element *E = identifier_map.front(); E; E = E->next()) {
  1155. rev_identifier_map[E->get()] = E->key();
  1156. }
  1157. Map<int, Variant> rev_constant_map;
  1158. const Variant *K = NULL;
  1159. while ((K = constant_map.next(K))) {
  1160. rev_constant_map[constant_map[*K]] = *K;
  1161. }
  1162. Map<int, uint32_t> rev_line_map;
  1163. for (Map<uint32_t, int>::Element *E = line_map.front(); E; E = E->next()) {
  1164. rev_line_map[E->get()] = E->key();
  1165. }
  1166. //save header
  1167. buf.resize(24);
  1168. buf.write[0] = 'G';
  1169. buf.write[1] = 'D';
  1170. buf.write[2] = 'S';
  1171. buf.write[3] = 'C';
  1172. encode_uint32(BYTECODE_VERSION, &buf.write[4]);
  1173. encode_uint32(identifier_map.size(), &buf.write[8]);
  1174. encode_uint32(constant_map.size(), &buf.write[12]);
  1175. encode_uint32(line_map.size(), &buf.write[16]);
  1176. encode_uint32(token_array.size(), &buf.write[20]);
  1177. //save identifiers
  1178. for (Map<int, StringName>::Element *E = rev_identifier_map.front(); E; E = E->next()) {
  1179. CharString cs = String(E->get()).utf8();
  1180. int len = cs.length() + 1;
  1181. int extra = 4 - (len % 4);
  1182. if (extra == 4)
  1183. extra = 0;
  1184. uint8_t ibuf[4];
  1185. encode_uint32(len + extra, ibuf);
  1186. for (int i = 0; i < 4; i++) {
  1187. buf.push_back(ibuf[i]);
  1188. }
  1189. for (int i = 0; i < len; i++) {
  1190. buf.push_back(cs[i] ^ 0xb6);
  1191. }
  1192. for (int i = 0; i < extra; i++) {
  1193. buf.push_back(0 ^ 0xb6);
  1194. }
  1195. }
  1196. for (Map<int, Variant>::Element *E = rev_constant_map.front(); E; E = E->next()) {
  1197. int len;
  1198. // Objects cannot be constant, never encode objects
  1199. Error err = encode_variant(E->get(), NULL, len, false);
  1200. ERR_FAIL_COND_V_MSG(err != OK, Vector<uint8_t>(), "Error when trying to encode Variant.");
  1201. int pos = buf.size();
  1202. buf.resize(pos + len);
  1203. encode_variant(E->get(), &buf.write[pos], len, false);
  1204. }
  1205. for (Map<int, uint32_t>::Element *E = rev_line_map.front(); E; E = E->next()) {
  1206. uint8_t ibuf[8];
  1207. encode_uint32(E->key(), &ibuf[0]);
  1208. encode_uint32(E->get(), &ibuf[4]);
  1209. for (int i = 0; i < 8; i++)
  1210. buf.push_back(ibuf[i]);
  1211. }
  1212. for (int i = 0; i < token_array.size(); i++) {
  1213. uint32_t token = token_array[i];
  1214. if (token & ~TOKEN_MASK) {
  1215. uint8_t buf4[4];
  1216. encode_uint32(token_array[i] | TOKEN_BYTE_MASK, &buf4[0]);
  1217. for (int j = 0; j < 4; j++) {
  1218. buf.push_back(buf4[j]);
  1219. }
  1220. } else {
  1221. buf.push_back(token);
  1222. }
  1223. }
  1224. return buf;
  1225. }
  1226. GDScriptTokenizerBuffer::Token GDScriptTokenizerBuffer::get_token(int p_offset) const {
  1227. int offset = token + p_offset;
  1228. if (offset < 0 || offset >= tokens.size())
  1229. return TK_EOF;
  1230. return GDScriptTokenizerBuffer::Token(tokens[offset] & TOKEN_MASK);
  1231. }
  1232. StringName GDScriptTokenizerBuffer::get_token_identifier(int p_offset) const {
  1233. int offset = token + p_offset;
  1234. ERR_FAIL_INDEX_V(offset, tokens.size(), StringName());
  1235. uint32_t identifier = tokens[offset] >> TOKEN_BITS;
  1236. ERR_FAIL_UNSIGNED_INDEX_V(identifier, (uint32_t)identifiers.size(), StringName());
  1237. return identifiers[identifier];
  1238. }
  1239. GDScriptFunctions::Function GDScriptTokenizerBuffer::get_token_built_in_func(int p_offset) const {
  1240. int offset = token + p_offset;
  1241. ERR_FAIL_INDEX_V(offset, tokens.size(), GDScriptFunctions::FUNC_MAX);
  1242. return GDScriptFunctions::Function(tokens[offset] >> TOKEN_BITS);
  1243. }
  1244. Variant::Type GDScriptTokenizerBuffer::get_token_type(int p_offset) const {
  1245. int offset = token + p_offset;
  1246. ERR_FAIL_INDEX_V(offset, tokens.size(), Variant::NIL);
  1247. return Variant::Type(tokens[offset] >> TOKEN_BITS);
  1248. }
  1249. int GDScriptTokenizerBuffer::get_token_line(int p_offset) const {
  1250. int offset = token + p_offset;
  1251. int pos = lines.find_nearest(offset);
  1252. if (pos < 0)
  1253. return -1;
  1254. if (pos >= lines.size())
  1255. pos = lines.size() - 1;
  1256. uint32_t l = lines.getv(pos);
  1257. return l & TOKEN_LINE_MASK;
  1258. }
  1259. int GDScriptTokenizerBuffer::get_token_column(int p_offset) const {
  1260. int offset = token + p_offset;
  1261. int pos = lines.find_nearest(offset);
  1262. if (pos < 0)
  1263. return -1;
  1264. if (pos >= lines.size())
  1265. pos = lines.size() - 1;
  1266. uint32_t l = lines.getv(pos);
  1267. return l >> TOKEN_LINE_BITS;
  1268. }
  1269. int GDScriptTokenizerBuffer::get_token_line_indent(int p_offset) const {
  1270. int offset = token + p_offset;
  1271. ERR_FAIL_INDEX_V(offset, tokens.size(), 0);
  1272. return tokens[offset] >> TOKEN_BITS;
  1273. }
  1274. const Variant &GDScriptTokenizerBuffer::get_token_constant(int p_offset) const {
  1275. int offset = token + p_offset;
  1276. ERR_FAIL_INDEX_V(offset, tokens.size(), nil);
  1277. uint32_t constant = tokens[offset] >> TOKEN_BITS;
  1278. ERR_FAIL_UNSIGNED_INDEX_V(constant, (uint32_t)constants.size(), nil);
  1279. return constants[constant];
  1280. }
  1281. String GDScriptTokenizerBuffer::get_token_error(int p_offset) const {
  1282. ERR_FAIL_V(String());
  1283. }
  1284. void GDScriptTokenizerBuffer::advance(int p_amount) {
  1285. ERR_FAIL_INDEX(p_amount + token, tokens.size());
  1286. token += p_amount;
  1287. }
  1288. GDScriptTokenizerBuffer::GDScriptTokenizerBuffer() {
  1289. token = 0;
  1290. }