godot/modules/gdscript/gd_tokenizer.cpp

/*************************************************************************/
/*  gd_tokenizer.cpp                                                     */
/*************************************************************************/
/*                       This file is part of:                           */
/*                           GODOT ENGINE                                */
/*                    http://www.godotengine.org                         */
/*************************************************************************/
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                 */
/*                                                                       */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the       */
/* "Software"), to deal in the Software without restriction, including   */
/* without limitation the rights to use, copy, modify, merge, publish,   */
/* distribute, sublicense, and/or sell copies of the Software, and to    */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions:                                             */
/*                                                                       */
/* The above copyright notice and this permission notice shall be        */
/* included in all copies or substantial portions of the Software.       */
/*                                                                       */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
/*************************************************************************/
#include "gd_tokenizer.h"
#include "print_string.h"
#include "gd_functions.h"
const char* GDTokenizer::token_names[TK_MAX]={
"Empty",
"Identifier",
"Constant",
"Self",
"Built-In Type",
"Built-In Func",
"In",
"'=='",
"'!='",
"'<'",
"'<='",
"'>'",
"'>='",
"'and'",
"'or'",
"'not'",
"'+'",
"'-'",
"'*'",
"'/'",
"'%'",
"'<<'",
"'>>'",
"'='",
"'+='",
"'-='",
"'*='",
"'/='",
"'%='",
"'<<='",
"'>>='",
"'&='",
"'|='",
"'^='",
"'&'",
"'|'",
"'^'",
"'~'",
//"Plus Plus",
//"Minus Minus",
"if",
"elif",
"else",
"for",
"do",
"while",
"switch",
"case",
"break",
"continue",
"pass",
"return",
"func",
"class",
"extends",
"tool",
"static",
"export",
"const",
"var",
"preload",
"assert",
"'['",
"']'",
"'{'",
"'}'",
"'('",
"')'",
"','",
"';'",
"'.'",
"'?'",
"':'",
"'\\n'",
"Error",
"EOF"};

const char *GDTokenizer::get_token_name(Token p_token) {

	ERR_FAIL_INDEX_V(p_token,TK_MAX,"<error>");
	return token_names[p_token];
}

static bool _is_text_char(CharType c) {

	return (c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='_';
}

static bool _is_number(CharType c) {

	return (c>='0' && c<='9');
}

static bool _is_hex(CharType c) {

	return (c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F');
}

void GDTokenizer::_make_token(Token p_type)  {

	TokenData &tk=tk_rb[tk_rb_pos];

	tk.type=p_type;
	tk.line=line;
	tk.col=column;

	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
}
void GDTokenizer::_make_identifier(const StringName& p_identifier)  {

	TokenData &tk=tk_rb[tk_rb_pos];

	tk.type=TK_IDENTIFIER;
	tk.identifier=p_identifier;
	tk.line=line;
	tk.col=column;

	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;

}

void GDTokenizer::_make_built_in_func(GDFunctions::Function p_func)  {

	TokenData &tk=tk_rb[tk_rb_pos];

	tk.type=TK_BUILT_IN_FUNC;
	tk.func=p_func;
	tk.line=line;
	tk.col=column;

	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;

}
void GDTokenizer::_make_constant(const Variant& p_constant)  {

	TokenData &tk=tk_rb[tk_rb_pos];

	tk.type=TK_CONSTANT;
	tk.constant=p_constant;
	tk.line=line;
	tk.col=column;

	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;

}

void GDTokenizer::_make_type(const Variant::Type& p_type) {


	TokenData &tk=tk_rb[tk_rb_pos];

	tk.type=TK_BUILT_IN_TYPE;
	tk.vtype=p_type;
	tk.line=line;
	tk.col=column;

	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;

}


void GDTokenizer::_make_error(const String& p_error) {

	error_flag=true;
	last_error=p_error;

	TokenData &tk=tk_rb[tk_rb_pos];
	tk.type=TK_ERROR;
	tk.constant=p_error;
	tk.line=line;
	tk.col=column;
	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;

}


void GDTokenizer::_make_newline(int p_spaces) {

	TokenData &tk=tk_rb[tk_rb_pos];
	tk.type=TK_NEWLINE;
	tk.constant=p_spaces;
	tk.line=line;
	tk.col=column;
	tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
}

void GDTokenizer::_advance() {

	if (error_flag) {
		//parser broke
		_make_error(last_error);
		return;
	}

	if (code_pos>=len) {
		_make_token(TK_EOF);
		return;
	}
#define GETCHAR(m_ofs) ((m_ofs+code_pos)>=len?0:_code[m_ofs+code_pos])
#define INCPOS(m_amount) { code_pos+=m_amount; column+=m_amount; }
	while (true) {


		bool is_node_path=false;

		switch(GETCHAR(0)) {
			case 0:
				_make_token(TK_EOF);
				break;
			case '\t':
			case '\r':
			case ' ':
				INCPOS(1);
				continue;
			case '\n': {
				line++;
				INCPOS(1);
				column=0;
				int i=0;
				while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
					i++;
				}

				_make_newline(i);
				return;
			}
#if 1 //py style tokenizer
			case '#': { // line comment skip

				while(GETCHAR(0)!='\n') {
					code_pos++;
					if (GETCHAR(0)==0) { //end of file
						_make_error("Unterminated Comment");
						return;
					}
				}
				INCPOS(1);
				column=0;
				line++;
				int i=0;
				while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
					i++;
				}
				_make_newline(i);
				return;

			} break;
#endif
			case '/': {

				switch(GETCHAR(1)) {
#if 0 // c style tokenizer
					case '*': { // block comment
						int pos = code_pos+2;
						int new_line=line;
						int new_col=column+2;

						while(true) {
							if (_code[pos]=='0') {
								_make_error("Unterminated Comment");
								code_pos=pos;
								return;
							}
							if (_code[pos]=='*' && _code[pos+1]=='/') {
								new_col+=2;
								pos+=2; //compensate
								break;
							} else if (_code[pos]=='\n') {
								new_line++;
								new_col=0;
							} else {
								new_col++;
							}
							pos++;
						}

						column=new_col;
						line=new_line;
						code_pos=pos;
						continue;

					} break;
					case '/': { // line comment skip

						while(GETCHAR(0)!='\n') {
							code_pos++;
							if (GETCHAR(0)==0) { //end of file
								_make_error("Unterminated Comment");
								return;
							}
						}
						INCPOS(1);
						column=0;
						line++;
						continue;

					} break;
#endif
					case '=': { // diveq

						_make_token(TK_OP_ASSIGN_DIV);
						INCPOS(1);

					} break;
					default:
						_make_token(TK_OP_DIV);

				}
			} break;
			case '=': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_EQUAL);
					INCPOS(1);

				} else
					_make_token(TK_OP_ASSIGN);

			} break;
			case '<': {
				if (GETCHAR(1)=='=') {

					_make_token(TK_OP_LESS_EQUAL);
					INCPOS(1);
				} else if (GETCHAR(1)=='<') {
					if (GETCHAR(2)=='=') {
						_make_token(TK_OP_ASSIGN_SHIFT_LEFT);
						INCPOS(1);
					} else {
						_make_token(TK_OP_SHIFT_LEFT);
					}
					INCPOS(1);
				} else
					_make_token(TK_OP_LESS);

			} break;
			case '>': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_GREATER_EQUAL);
					INCPOS(1);
				} else if (GETCHAR(1)=='>') {
					if (GETCHAR(2)=='=') {
						_make_token(TK_OP_ASSIGN_SHIFT_RIGHT);
						INCPOS(1);

					} else {
						_make_token(TK_OP_SHIFT_RIGHT);
					}
					INCPOS(1);
				} else {
					_make_token(TK_OP_GREATER);
				}

			} break;
			case '!': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_NOT_EQUAL);
					INCPOS(1);
				} else {
					_make_token(TK_OP_NOT);
				}

			} break;
			//case '"' //string - no strings in shader
			//case '\'' //string - no strings in shader
			case '{':
				_make_token(TK_CURLY_BRACKET_OPEN);
				break;
			case '}':
				_make_token(TK_CURLY_BRACKET_CLOSE);
				break;
			case '[':
				_make_token(TK_BRACKET_OPEN);
				break;
			case ']':
				_make_token(TK_BRACKET_CLOSE);
				break;
			case '(':
				_make_token(TK_PARENTHESIS_OPEN);
				break;
			case ')':
				_make_token(TK_PARENTHESIS_CLOSE);
				break;
			case ',':
				_make_token(TK_COMMA);
				break;
			case ';':
				_make_token(TK_SEMICOLON);
				break;
			case '?':
				_make_token(TK_QUESTION_MARK);
				break;
			case ':':
				_make_token(TK_COLON); //for methods maybe but now useless.
				break;
			case '^': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_BIT_XOR);
					INCPOS(1);
				} else {
					_make_token(TK_OP_BIT_XOR);
				}

			} break;
			case '~':
				_make_token(TK_OP_BIT_INVERT);
				break;
			case '&': {
				if (GETCHAR(1)=='&') {

					_make_token(TK_OP_AND);
					INCPOS(1);
				} else if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_BIT_AND);
					INCPOS(1);
				} else {
					_make_token(TK_OP_BIT_AND);
				}
			} break;
			case '|': {
				if (GETCHAR(1)=='|') {

					_make_token(TK_OP_OR);
					INCPOS(1);
				} else if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_BIT_OR);
					INCPOS(1);
				} else {
					_make_token(TK_OP_BIT_OR);
				}
			} break;
			case '*': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_MUL);
					INCPOS(1);
				} else {
					_make_token(TK_OP_MUL);
				}
			} break;
			case '+': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_ADD);
					INCPOS(1);
				//}  else if (GETCHAR(1)=='+') {
				//	_make_token(TK_OP_PLUS_PLUS);
				//	INCPOS(1);
				} else {
					_make_token(TK_OP_ADD);
				}

			} break;
			case '-': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_SUB);
					INCPOS(1);
				//}  else if (GETCHAR(1)=='-') {
				//	_make_token(TK_OP_MINUS_MINUS);
				//	INCPOS(1);
				} else {
					_make_token(TK_OP_SUB);
				}
			} break;
			case '%': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_MOD);
					INCPOS(1);
				} else {
					_make_token(TK_OP_MOD);
				}
			} break;
			case '@':
				if (CharType(GETCHAR(1))!='"') {
					_make_error("Unexpected '@'");
					return;
				}
				INCPOS(1);
				is_node_path=true;
			case '"': {

				int i=1;
				String str;
				while(true) {
					if (CharType(GETCHAR(i)==0)) {

						_make_error("Unterminated String");
						return;
					} else if (CharType(GETCHAR(i)=='"')) {
						break;
					} else if (CharType(GETCHAR(i)=='\\')) {
						//escaped characters...
						i++;
						CharType next = GETCHAR(i);
						if (next==0) {
							_make_error("Unterminated String");
							return;
						}
						CharType res=0;

						switch(next) {

							case 'a': res=7; break;
							case 'b': res=8; break;
							case 't': res=9; break;
							case 'n': res=10; break;
							case 'v': res=11; break;
							case 'f': res=12; break;
							case 'r': res=13; break;
							case '\'': res='\''; break;
							case '\"': res='\"'; break;
							case '\\': res='\\'; break;
							case 'x': {
								//hexnumbarh - oct is deprecated

								int read=0;
								for(int j=0;j<4;j++) {
									CharType c = GETCHAR(i+j);
									if (c==0) {
										_make_error("Unterminated String");
										return;
									}
									if (!_is_hex(c)) {
										if (j==0 || !(j&1)) {
											_make_error("Malformed hex constant in string");
											return;
										} else
											break;
									}
									CharType v;
									if (c>='0' && c<='9') {
										v=c-'0';
									} else if (c>='a' && c<='f') {
										v=c-'a';
										v+=10;
									} else if (c>='A' && c<='F') {
										v=c-'A';
										v+=10;
									} else {
										ERR_PRINT("BUG");
										v=0;
									}

									res<<=4;
									res|=v;

									read++;
								}
								i+=read-1;


							} break;
							default: {

								_make_error("Invalid escape sequence");
								return;
							} break;
						}

						str+=res;

					} else {
						str+=CharType(GETCHAR(i));
					}
					i++;
				}
				INCPOS(i);

				if (is_node_path) {
					_make_constant(NodePath(str));
				} else {
					_make_constant(str);
				}

			} break;
			default: {

				if (_is_number(GETCHAR(0)) || (GETCHAR(0)=='.' && _is_number(GETCHAR(1)))) {
					// parse number
					bool period_found=false;
					bool exponent_found=false;
					bool hexa_found=false;
					bool sign_found=false;

					String str;
					int i=0;

					while(true) {
						if (GETCHAR(i)=='.') {
							if (period_found || exponent_found) {
                                _make_error("Invalid numeric constant at '.'");
								return;
							}
							period_found=true;
						} else if (GETCHAR(i)=='x') {
                            if (hexa_found || str.length()!=1 || !( (i==1 && str[0]=='0') || (i==2 && str[1]=='0' && str[0]=='-') ) ) {
                                _make_error("Invalid numeric constant at 'x'");
								return;
							}
							hexa_found=true;
                        } else if (!hexa_found && GETCHAR(i)=='e') {
							if (hexa_found || exponent_found) {
                                _make_error("Invalid numeric constant at 'e'");
								return;
							}
							exponent_found=true;
						} else if (_is_number(GETCHAR(i))) {
							//all ok
						} else if (hexa_found && _is_hex(GETCHAR(i))) {

						} else if ((GETCHAR(i)=='-' || GETCHAR(i)=='+') && exponent_found) {
							if (sign_found) {
                                _make_error("Invalid numeric constant at '-'");
								return;
							}
							sign_found=true;
						} else
							break;

						str+=CharType(GETCHAR(i));
						i++;
					}

                    if (!( _is_number(str[str.length()-1]) || (hexa_found && _is_hex(str[str.length()-1])))) {
                        _make_error("Invalid numeric constant: "+str);
						return;
					}

					INCPOS(str.length());
                    if (hexa_found) {
                        int val = str.hex_to_int();
                        _make_constant(val);
                    } else if (period_found) {
						real_t val = str.to_double();
						//print_line("*%*%*%*% to convert: "+str+" result: "+rtos(val));
						_make_constant(val);
                    } else {
						int val = str.to_int();
						_make_constant(val);

					}

					return;
				}

				if (GETCHAR(0)=='.') {
					//parse period
					_make_token(TK_PERIOD);
					break;
				}

				if (_is_text_char(GETCHAR(0))) {
					// parse identifier
					String str;
					str+=CharType(GETCHAR(0));

					int i=1;
					while(_is_text_char(GETCHAR(i))) {
						str+=CharType(GETCHAR(i));
						i++;
					}

					bool identifier=false;

					if (str=="null") {
						_make_constant(Variant());

					} else if (str=="true") {
						_make_constant(true);

					} else if (str=="false") {
						_make_constant(false);
					} else {

						bool found=false;

						struct _bit { Variant::Type type; const char *text;};
						//built in types

						static const  _bit type_list[]={
							//types
							{Variant::BOOL,"bool"},
							{Variant::INT,"int"},
							{Variant::REAL,"float"},
							{Variant::STRING,"String"},
							{Variant::VECTOR2,"vec2"},
							{Variant::VECTOR2,"Vector2"},
							{Variant::RECT2,"Rect2"},
							{Variant::MATRIX32,"Matrix32"},
							{Variant::MATRIX32,"mat32"},
							{Variant::VECTOR3,"vec3"},
							{Variant::VECTOR3,"Vector3"},
							{Variant::_AABB,"AABB"},
							{Variant::_AABB,"Rect3"},
							{Variant::PLANE,"Plane"},
							{Variant::QUAT,"Quat"},
							{Variant::MATRIX3,"mat3"},
							{Variant::MATRIX3,"Matrix3"},
							{Variant::TRANSFORM,"trn"},
							{Variant::TRANSFORM,"Transform"},
							{Variant::COLOR,"Color"},
							{Variant::IMAGE,"Image"},
							{Variant::_RID,"RID"},
							{Variant::OBJECT,"Object"},
							{Variant::INPUT_EVENT,"InputEvent"},
							{Variant::DICTIONARY,"dict"},
							{Variant::DICTIONARY,"Dictionary"},
							{Variant::ARRAY,"Array"},
							{Variant::RAW_ARRAY,"RawArray"},
							{Variant::INT_ARRAY,"IntArray"},
							{Variant::REAL_ARRAY,"FloatArray"},
							{Variant::STRING_ARRAY,"StringArray"},
							{Variant::VECTOR2_ARRAY,"Vector2Array"},
							{Variant::VECTOR3_ARRAY,"Vector3Array"},
							{Variant::COLOR_ARRAY,"ColorArray"},
							{Variant::VARIANT_MAX,NULL},
						};

						{


							int idx=0;

							while(type_list[idx].text) {

								if (str==type_list[idx].text) {
									_make_type(type_list[idx].type);
									found=true;
									break;
								}
								idx++;
							}
						}

						if (!found) {

							//built in func?

							for(int i=0;i<GDFunctions::FUNC_MAX;i++) {

								if (str==GDFunctions::get_func_name(GDFunctions::Function(i))) {

									_make_built_in_func(GDFunctions::Function(i));
									found=true;
									 break;
								}
							}

							//keywor
						}

						if (!found) {


							struct _kws { Token token; const char *text;};

							static const  _kws keyword_list[]={
								//ops
								{TK_OP_IN,"in"},
								{TK_OP_NOT,"not"},
								{TK_OP_OR,"or"},
								{TK_OP_AND,"and"},
								//func
								{TK_PR_FUNCTION,"func"},
								{TK_PR_FUNCTION,"function"},
								{TK_PR_CLASS,"class"},
								{TK_PR_EXTENDS,"extends"},
								{TK_PR_TOOL,"tool"},
								{TK_PR_STATIC,"static"},
								{TK_PR_EXPORT,"export"},
								{TK_PR_VAR,"var"},
								{TK_PR_PRELOAD,"preload"},
								{TK_PR_ASSERT,"assert"},
								{TK_PR_CONST,"const"},
								//controlflow
								{TK_CF_IF,"if"},
								{TK_CF_ELIF,"elif"},
								{TK_CF_ELSE,"else"},
								{TK_CF_FOR,"for"},
								{TK_CF_WHILE,"while"},
								{TK_CF_DO,"do"},
								{TK_CF_SWITCH,"switch"},
								{TK_CF_BREAK,"break"},
								{TK_CF_CONTINUE,"continue"},
								{TK_CF_RETURN,"return"},
								{TK_CF_PASS,"pass"},
								{TK_SELF,"self"},
								{TK_ERROR,NULL}
							};

							int idx=0;
							found=false;

							while(keyword_list[idx].text) {

								if (str==keyword_list[idx].text) {
									_make_token(keyword_list[idx].token);
									found=true;
									break;
								}
								idx++;
							}
						}

						if (!found)
							identifier=true;
					}


					if (identifier) {
						_make_identifier(str);
					}
					INCPOS(str.length());
					return;
				}

				_make_error("Unknown character");
				return;

			} break;
		}

		INCPOS(1);
		break;
	}

}

void GDTokenizer::set_code(const String& p_code) {

	code=p_code;
	len = p_code.length();
	if (len) {
		_code=&code[0];
	} else {
		_code=NULL;
	}
	code_pos=0;
	line=1; //it is stand-ar-ized that lines begin in 1 in code..
	column=0;
	tk_rb_pos=0;
	error_flag=false;
	last_error="";
	for(int i=0;i<MAX_LOOKAHEAD+1;i++)
		_advance();
}

GDTokenizer::Token GDTokenizer::get_token(int p_offset) const {
	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, TK_ERROR);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, TK_ERROR);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	return tk_rb[ofs].type;
}

int GDTokenizer::get_token_line(int p_offset) const {
	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	return tk_rb[ofs].line;
}

int GDTokenizer::get_token_column(int p_offset) const {
	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	return tk_rb[ofs].col;
}

const Variant& GDTokenizer::get_token_constant(int p_offset) const {
	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, tk_rb[0].constant);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, tk_rb[0].constant);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_CONSTANT,tk_rb[0].constant);
	return tk_rb[ofs].constant;
}
StringName GDTokenizer::get_token_identifier(int p_offset) const {

	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, StringName());
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, StringName());

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_IDENTIFIER,StringName());
	return tk_rb[ofs].identifier;

}

GDFunctions::Function GDTokenizer::get_token_built_in_func(int p_offset) const {

	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, GDFunctions::FUNC_MAX);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, GDFunctions::FUNC_MAX);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_FUNC,GDFunctions::FUNC_MAX);
	return tk_rb[ofs].func;

}

Variant::Type GDTokenizer::get_token_type(int p_offset) const {

	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, Variant::NIL);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, Variant::NIL);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_TYPE,Variant::NIL);
	return tk_rb[ofs].vtype;

}


int GDTokenizer::get_token_line_indent(int p_offset) const {

	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, 0);
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, 0);

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_NEWLINE,0);
	return tk_rb[ofs].constant;

}

String GDTokenizer::get_token_error(int p_offset) const {

	ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, String());
	ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, String());

	int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
	ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_ERROR,String());
	return tk_rb[ofs].constant;
}

void GDTokenizer::advance(int p_amount) {

	ERR_FAIL_COND( p_amount <=0 );
	for(int i=0;i<p_amount;i++)
		_advance();
}