Add an option to the lexer to return comments and use it by scripts

Also added some keywords to allow compilation of thinscript

SquiLu/squirrel/sqcompiler.cpp

@@ -165,7 +165,7 @@ public:
 		c->Error(s);
 	}
 
-	void Error(const SQChar *s, ...)
+	void Error(const SQChar *s, ...) //__attribute__((format(printf, 2, 3)))
 	{
 		va_list vl;
 		va_start(vl, s);
@@ -175,7 +175,7 @@ public:
 		longjmp(_errorjmp,1);
 	}
 
-	void Warning(const SQChar *s, ...)
+	void Warning(const SQChar *s, ...) //__attribute__((format(printf, 2, 3)))
 	{
 	    if(!_show_warnings) return;
 		va_list vl;
@@ -660,6 +660,11 @@ public:
 			{
 			Lex();
 			id = Expect(TK_IDENTIFIER);
+			if(_token == _SC(':')) {
+                //type specifier like typescript
+                Lex();
+                ExpectTypeToken(); //ignore for now
+			}
 			Expect('=');
 			SQObjectPtr strongid = id;
 			CheckLocalNameScope(id, _scope.nested);
@@ -683,6 +688,7 @@ public:
             Lex(); //ignore it only to allow run some C/C++ code
             goto start_again;
 
+        case TK_DECLARE: //parse as extern
         case TK_EXTERN: ExternDeclStatement();	break;
 
         case TK_TYPEDEF:
@@ -713,6 +719,11 @@ public:
 		    Pragma();
 		    break;
 
+		case TK_UNSAFE:
+		    Lex(); //ignore for now
+		    goto start_again;
+		    break;
+
         case TK_IDENTIFIER:
             id = _fs->CreateString(_lex._svalue);
             if(CheckTypeName(id)) //C/C++ type declaration;
@@ -1148,6 +1159,13 @@ public:
 				Lex();
 				FunctionCallArgs();
 				break;
+
+            case TK_AS:
+            {
+                Lex();
+                ExpectTypeToken(); //ignore for now
+            }
+
 			default: return;
 			}
 		}
@@ -1579,9 +1597,13 @@ function_params_decl:
 			//-1 to compensate default parameters when relocating
 			CreateFunction(varname, eFunctionType::local, -1);
 			if(_is_parsing_extern) {
-                Expect(_SC(';'));
-                CheckExternName(varname, true);
-                return;
+                if(_token == _SC(';')) //to parse thinscript
+                {
+                    //Expect(_SC(';'));
+                    CheckExternName(varname, true);
+                    return;
+                }
+                else _is_parsing_extern = false;
 			}
 			_fs->AddInstruction(_OP_CLOSURE, _fs->PushTarget(), _fs->_functions.size() - 1, 0);
 			//rellocate any stack operation (default parameters & _OP_Closure)
@@ -1611,6 +1633,11 @@ function_params_decl:
                 Lex();
                 goto function_params_decl;
 			}
+			else if(_token == _SC(':')) {
+                //type specifier like typescript
+                Lex();
+                ExpectTypeToken(); //ignore for now
+			}
             if(is_void_declaration)
             {
                 Error(_SC("void type is invalid here"));
@@ -2326,8 +2353,12 @@ error:
 		}
 
         if(_is_parsing_extern) {
-            _fs->PopChildState();
-            return;
+            if(_token == _SC(';')) //to parse thinscript
+            {
+                _fs->PopChildState();
+                return;
+            }
+            else _is_parsing_extern = false;
         }
 
 		SQFuncState *currchunk = _fs;

SquiLu/squirrel/sqcompiler.h

@@ -8,6 +8,7 @@ struct SQVM;
     ENUM_TK(3WAYSCMP)\
     ENUM_TK(AND)\
     ENUM_TK(ARROW)\
+    ENUM_TK(AS)\
     ENUM_TK(ATTR_CLOSE)\
     ENUM_TK(ATTR_OPEN)\
     ENUM_TK(BASE)\
@@ -22,8 +23,11 @@ struct SQVM;
     ENUM_TK(CLASS)\
     ENUM_TK(CLONE)\
     ENUM_TK(CONST)\
+    ENUM_TK(COMMENT_BLOCK)\
+    ENUM_TK(COMMENT_LINE)\
     ENUM_TK(CONSTRUCTOR)\
     ENUM_TK(CONTINUE)\
+    ENUM_TK(DECLARE)\
     ENUM_TK(DEFAULT)\
     ENUM_TK(DELETE)\
     ENUM_TK(DESTRUCTOR)\
@@ -110,6 +114,7 @@ struct SQVM;
     ENUM_TK(TRY)\
     ENUM_TK(TYPEOF)\
     ENUM_TK(UMINUS)\
+    ENUM_TK(UNSAFE)\
     ENUM_TK(USING)\
     ENUM_TK(USHIFTR)\
     ENUM_TK(VARPARAMS)\

SquiLu/squirrel/sqlexer.cpp

@@ -25,8 +25,10 @@ SQLexer::~SQLexer()
 	_keywords->Release();
 }
 
-SQInteger SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up,CompilerErrorFunc efunc,void *ed)
+SQInteger SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg,
+                        SQUserPointer up,CompilerErrorFunc efunc,void *ed, SQBool want_comments)
 {
+    _want_comments = want_comments;
     _lasterror[0] = '\0';
     _svalue = NULL;
 	_errfunc = efunc;
@@ -53,6 +55,7 @@ SQTable * SQLexer::GetKeywords()
 	SQTable *tbl = SQTable::Create(_sharedstate, (TK_LAST_ENUM_TOKEN - TK_FIRST_ENUM_TOKEN - 1) /*26*/);
 	ADD_KEYWORD(any_t, TK_LOCAL_ANY_T);
 	ADD_KEYWORD(array_t, TK_LOCAL_ARRAY_T);
+	ADD_KEYWORD(as, TK_AS);
 	ADD_KEYWORD(auto, TK_LOCAL);
 	ADD_KEYWORD(base, TK_BASE);
 	ADD_KEYWORD(bool_t, TK_LOCAL_BOOL_T);
@@ -65,6 +68,7 @@ SQTable * SQLexer::GetKeywords()
 	ADD_KEYWORD(constructor,TK_CONSTRUCTOR);
 	ADD_KEYWORD(const,TK_CONST);
 	ADD_KEYWORD(continue, TK_CONTINUE);
+	ADD_KEYWORD(declare, TK_DECLARE);
 	ADD_KEYWORD(default, TK_DEFAULT);
 	ADD_KEYWORD(delete, TK_DELETE);
 	ADD_KEYWORD(destructor,TK_DESTRUCTOR);
@@ -121,6 +125,7 @@ SQTable * SQLexer::GetKeywords()
 	ADD_KEYWORD(uint64_t, TK_LOCAL_UINT64_T);
 	ADD_KEYWORD(uint8_t, TK_LOCAL_UINT8_T);
 	ADD_KEYWORD(uint_t, TK_LOCAL_UINT_T);
+	ADD_KEYWORD(unsafe, TK_UNSAFE);
 	ADD_KEYWORD(using, TK_USING);
 	ADD_KEYWORD(var, TK_LOCAL);
 	ADD_KEYWORD(virtual, TK_VIRTUAL);
@@ -178,7 +183,7 @@ const SQChar *SQLexer::GetTokenName(int tk_code) {
         SQ_KEYWORDS_LIST()
 #undef ENUM_TK
         default:
-            str_tk = _SC("???");
+            str_tk = _SC("()");
     }
     return str_tk;
 }
@@ -196,19 +201,35 @@ SQInteger SQLexer::LexBlockComment()
     }
 */
 	bool done = false;
+	if(_want_comments) INIT_TEMP_STRING();
+	NEXT(); //remove the comment token '*'
 	while(!done) {
 		switch(CUR_CHAR) {
-			case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); }}; continue;
-			case _SC('\n'): _currentline++; NEXT(); continue;
+			case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); continue;}}; break;
+			case _SC('\n'): _currentline++; break;
 			case SQUIRREL_EOB: return Error(_SC("missing \"*/\" in comment"));
-			default: NEXT();
 		}
+		if(_want_comments) APPEND_CHAR(CUR_CHAR);
+		NEXT();
 	}
+    if(_want_comments)
+    {
+        TERMINATE_BUFFER();
+        if(_longstr.size() > 0) _longstr.pop_back(); //remove the last '*'
+        _svalue = &_longstr[0];
+    }
 	return 0;
 }
 SQInteger SQLexer::LexLineComment()
 {
-	do { NEXT(); } while (CUR_CHAR != _SC('\n') && (!IS_EOB()));
+    if(_want_comments) INIT_TEMP_STRING();
+    NEXT(); //remove the comment token
+	while (CUR_CHAR != _SC('\n') && (!IS_EOB())) {if(_want_comments) APPEND_CHAR(CUR_CHAR); NEXT();}
+    if(_want_comments)
+    {
+        TERMINATE_BUFFER();
+        _svalue = &_longstr[0];
+    }
 	return 0;
 }
 
@@ -230,6 +251,7 @@ SQInteger SQLexer::Lex()
 		    if(CUR_CHAR == '!') //shell shebang
             {
                 if(LexLineComment()) return -1;
+                if(_want_comments) RETURN_TOKEN(TK_COMMENT_LINE)
                 continue;
             }
             RETURN_TOKEN(TK_PRAGMA);
@@ -238,11 +260,12 @@ SQInteger SQLexer::Lex()
 			NEXT();
 			switch(CUR_CHAR){
 			case _SC('*'):
-				NEXT();
 				if(LexBlockComment()) return -1;
+                if(_want_comments) RETURN_TOKEN(TK_COMMENT_BLOCK)
 				continue;
 			case _SC('/'):
 				if(LexLineComment()) return -1;
+                if(_want_comments) RETURN_TOKEN(TK_COMMENT_LINE)
 				continue;
 			case _SC('='):
 				NEXT();

SquiLu/squirrel/sqlexer.h

@@ -12,7 +12,8 @@ struct SQLexer
 {
 	SQLexer();
 	virtual ~SQLexer();
-	SQInteger Init(SQSharedState *ss,SQLEXREADFUNC rg,SQUserPointer up,CompilerErrorFunc efunc,void *ed);
+	SQInteger Init(SQSharedState *ss,SQLEXREADFUNC rg,SQUserPointer up,
+                CompilerErrorFunc efunc,void *ed, SQBool want_comments=SQFalse);
 	SQInteger ResetReader(SQLEXREADFUNC rg, SQUserPointer up, SQInteger line);
 	SQTable * GetKeywords();
 	SQInteger Error(const SQChar *err, ...);
@@ -54,6 +55,7 @@ public:
 	CompilerErrorFunc _errfunc;
 	void *_errtarget;
 	SQChar _lasterror[256];
+	SQBool _want_comments;
 };
 
 #endif