BansheeSL: Parser/Lexer working with no code blocks

BansheeSL/BsLexerFX.h

@@ -354,6 +354,10 @@ YYSTYPE * yyget_lval (yyscan_t yyscanner );
 
 void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
 
+       YYLTYPE *yyget_lloc (yyscan_t yyscanner );
+    
+        void yyset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner );
+    
 /* %endif */
 
 /* Macros after this point can all be overridden by user definitions in
@@ -414,10 +418,10 @@ static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
 /* %if-c-only Standard (non-C++) definition */
 
 extern int yylex \
-               (YYSTYPE * yylval_param ,yyscan_t yyscanner);
+               (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner);
 
 #define YY_DECL int yylex \
-               (YYSTYPE * yylval_param , yyscan_t yyscanner)
+               (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner)
 /* %endif */
 /* %if-c++-only C++ definition */
 /* %endif */
@@ -447,8 +451,8 @@ extern int yylex \
 #undef YY_DECL
 #endif
 
-#line 220 "BsLexerFX.l"
+#line 226 "BsLexerFX.l"
 
-#line 453 "BsLexerFX.h"
+#line 457 "BsLexerFX.h"
 #undef yyIN_HEADER
 #endif /* yyHEADER_H */

BansheeSL/BsLexerFX.l

@@ -1,23 +1,28 @@
 %{
 #include "BsParserFX.h"
+
+#define YY_USER_ACTION yylloc->first_column = yycolumn + 1; yylloc->first_line = yylineno + 1; yycolumn += (int)yyleng;
+#define YY_USER_INIT yylineno = 0; yycolumn = 0;
 %}
  
-%option reentrant noyywrap nounistd never-interactive warn nodefault bison-bridge
+%option yylineno reentrant noyywrap nounistd never-interactive warn nodefault bison-bridge bison-locations
 %option outfile="BsLexerFX.c" header-file="BsLexerFX.h"
 %option extra-type="struct tagParseState *"
 %option debug
 
 INTEGER			[0-9][0-9]*
-INTEGER_BIN		0x[0-9][0-9]*
+INTEGER_16		0[xX][0-9a-fA-F]+
 FLOAT			[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?[fF]?
 STRING			\"[^"\n]*\"
 IDENTIFIER		[_a-zA-Z][_a-zA-Z0-9]*
 WS				[ \r\n\t]*
+COMMENT			\/\/[^\n]*
 
 %%
 
 {WS}			{ /* Skip blank */ }
 {INTEGER}       { yylval->intValue = atoi(yytext); return TOKEN_INTEGER; }
+{INTEGER_16}    { yylval->intValue = (int)strtol(yytext, 0, 0); return TOKEN_INTEGER; }
 {FLOAT}			{ yylval->floatValue = (float)atof(yytext); return TOKEN_FLOAT; }
 {STRING}		{ yylval->strValue = mmalloc_strdup(yyextra->memContext, yytext); return TOKEN_STRING; }
 true			{ yylval->intValue = 1; return TOKEN_BOOLEAN; }
@@ -216,5 +221,6 @@ anisotropiccmp	{ yylval->intValue = 6; return TOKEN_FILTERVALUE; }
 static			{ yylval->intValue = 0; return TOKEN_BUFFERUSAGE; }
 dynamic			{ yylval->intValue = 1; return TOKEN_BUFFERUSAGE; }
 
+{COMMENT}		{ }
 {IDENTIFIER}	{ yylval->strValue = mmalloc_strdup(yyextra->memContext, yytext); return TOKEN_IDENTIFIER; }
 .				{ return yytext[0]; }

BansheeSL/BsParserFX.h

@@ -41,7 +41,7 @@ extern int yydebug;
 #endif
 /* "%code requires" blocks.  */
 /* Line 2579 of glr.c  */
-#line 13 "BsParserFX.y"
+#line 9 "BsParserFX.y"
 
 #include "BsMMAlloc.h"
 #include "BsASTFX.h"
@@ -190,7 +190,7 @@ extern int yydebug;
 typedef union YYSTYPE
 {
 /* Line 2579 of glr.c  */
-#line 49 "BsParserFX.y"
+#line 46 "BsParserFX.y"
 
 	int intValue;
 	float floatValue;
@@ -208,6 +208,19 @@ typedef union YYSTYPE
 # define YYSTYPE_IS_DECLARED 1
 #endif
 
+#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
+typedef struct YYLTYPE
+{
+  int first_line;
+  int first_column;
+  int last_line;
+  int last_column;
+} YYLTYPE;
+# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
+# define YYLTYPE_IS_DECLARED 1
+# define YYLTYPE_IS_TRIVIAL 1
+#endif
+
 
 int yyparse (ParseState* parse_state, yyscan_t scanner);
 

BansheeSL/BsParserFX.y

@@ -3,11 +3,7 @@
 #include "BsLexerFX.h"
 #define inline
 
-	void yyerror(ParseState* parse_state, yyscan_t scanner, const char *msg) 
-	{ 
-		fprintf (stderr, "%s\n", msg);
-	}
-
+void yyerror(YYLTYPE *locp, ParseState* parse_state, yyscan_t scanner, const char *msg);
 %}
 
 %code requires{
@@ -41,6 +37,7 @@
 
 %define api.pure
 %debug
+%locations
 %lex-param { yyscan_t scanner }
 %parse-param { ParseState* parse_state }
 %parse-param { yyscan_t scanner }
@@ -343,7 +340,7 @@ stencil_op_body
 stencil_op_option
 	: TOKEN_FAIL '=' TOKEN_OPVALUE ';'					{ $$.type = OT_Fail; $$.value.intValue = $3; }
 	| TOKEN_ZFAIL '=' TOKEN_OPVALUE ';'					{ $$.type = OT_ZFail; $$.value.intValue = $3; }
-	| TOKEN_PASS '=' TOKEN_OPVALUE ';'					{ $$.type = OT_Pass; $$.value.intValue = $3; }
+	| TOKEN_PASS '=' TOKEN_OPVALUE ';'					{ $$.type = OT_PassOp; $$.value.intValue = $3; }
 	| TOKEN_COMPAREFUNC '=' TOKEN_COMPFUNCVALUE ';'		{ $$.type = OT_CompareFunc; $$.value.intValue = $3; }
 	; 
 
@@ -742,9 +739,14 @@ qualifier_list
 	;
 
 qualifier
-	: ':' TOKEN_ALIAS '(' TOKEN_IDENTIFIER ')'	{ $$.type = OT_Alias; $$.value.strValue = $4; }
+	: ':' TOKEN_ALIAS '(' TOKEN_STRING ')'		{ $$.type = OT_Alias; $$.value.strValue = $4; }
 	| ':' TOKEN_AUTO '(' TOKEN_STRING ')'		{ $$.type = OT_Auto; $$.value.strValue = $4; }
 	| ':' TOKEN_SHARED '(' TOKEN_BOOLEAN ')'	{ $$.type = OT_Shared; $$.value.intValue = $4; }
 	| ':' TOKEN_USAGE '(' TOKEN_BUFFERUSAGE ')'	{ $$.type = OT_Usage; $$.value.intValue = $4; }
 	;
-%%
+%%
+
+void yyerror(YYLTYPE *locp, ParseState* parse_state, yyscan_t scanner, const char *msg) 
+{ 
+	fprintf (stderr, "%s -- Line: %i Column: %i\n", msg, locp->first_line, locp->first_column);
+}

BansheeSL/Include/BsASTFX.h

@@ -59,6 +59,7 @@ enum tagOptionType
 	OT_StencilWriteMask,
 	OT_StencilOpFront,
 	OT_StencilOpBack,
+	OT_PassOp,
 	OT_Fail,
 	OT_ZFail,
 	OT_AlphaToCoverage,

BansheeSL/Source/BSMMAlloc.c

@@ -36,6 +36,9 @@ void* mmalloc(void* context, int size)
 	MMAllocHeader* parent = (MMAllocHeader*)context;
 
 	header->next = parent->next;
+	if (parent->next)
+		parent->next->prev = header;
+
 	header->prev = parent;
 	parent->next = header;
 

BansheeSL/Source/BsASTFX.c

@@ -28,6 +28,7 @@ OptionInfo OPTION_LOOKUP[] =
 	{ OT_StencilWriteMask, ODT_Int },
 	{ OT_StencilOpFront, ODT_Complex },
 	{ OT_StencilOpBack, ODT_Complex },
+	{ OT_PassOp, ODT_Int },
 	{ OT_Fail, ODT_Int },
 	{ OT_ZFail, ODT_Int },
 	{ OT_AlphaToCoverage, ODT_Bool },
@@ -129,9 +130,6 @@ void nodeOptionsResize(void* context, NodeOptions* options, int size)
 	memcpy(options->entries, originalEntries, sizeToCopy);
 	memset(options->entries + elementsToCopy, 0, sizeof(NodeOption) * options->bufferSize - sizeToCopy);
 
-	for (i = 0; i < originalCount; i++)
-		nodeOptionDelete(&originalEntries[i]);
-
 	mmfree(originalEntries);
 }
 

BansheeSL/Source/BsSLPlugin.cpp

@@ -3,6 +3,10 @@
 #include "BsFileSystem.h"
 #include "BsDataStream.h"
 
+// DEBUG ONLY
+#include "BsDebug.h"
+#include "BsMatrix4.h"
+
 extern "C" {
 #include "BsMMAlloc.h"
 #include "BsASTFX.h"
@@ -42,6 +46,47 @@ namespace BansheeEngine
 		return SystemName;
 	}
 
+	void debugPrint(ASTFXNode* node, String indent)
+	{
+		LOGWRN(indent + "NODE " + toString(node->type));
+
+		for (int i = 0; i < node->options->count; i++)
+		{
+			OptionDataType odt = OPTION_LOOKUP[(int)node->options->entries[i].type].dataType;
+			if (odt == ODT_Complex)
+			{
+				LOGWRN(indent + toString(i) + ". " + toString(node->options->entries[i].type));
+				debugPrint(node->options->entries[i].value.nodePtr, indent + "\t");
+				continue;
+			}
+
+			String value;
+			switch (odt)
+			{
+			case ODT_Bool:
+				value = toString(node->options->entries[i].value.intValue != 0);
+				break;
+			case ODT_Int:
+				value = toString(node->options->entries[i].value.intValue);
+				break;
+			case ODT_Float:
+				value = toString(node->options->entries[i].value.floatValue);
+				break;
+			case ODT_String:
+				value = node->options->entries[i].value.strValue;
+				break;
+			case ODT_Matrix:
+				{
+					Matrix4 mat4 = *(Matrix4*)(node->options->entries[i].value.matrixValue);
+					value = toString(mat4);
+				}
+				break;
+			}
+
+			LOGWRN(indent + toString(i) + ". " + toString(node->options->entries[i].type) + " = " + value);
+		}
+	}
+
 	/**
 	 * @brief	Entry point to the plugin. Called by the engine when the plugin is loaded.
 	 */
@@ -59,6 +104,7 @@ namespace BansheeEngine
 		ParseState* parseState = parseStateCreate();
 		parseFX(parseState, contents.c_str());
 
+		debugPrint(parseState->rootNode, "");
 		int bp = 0;
 
 		parseStateDelete(parseState);

TODO.txt

@@ -229,16 +229,22 @@ Compute = code
 -------------------
 
 TODO:
- - BorderColor for SamplerState isn't implemented yet as I haven't decided how to deal with vectors/matrices yet
- - Parser logic for parameters and blocks (with qualifier support)
+ - Test the example shader and ensure parameters/blocks/qualifiers/default values and nested nodes work properly
+ - Stencil read and write masks don't accept binary masks but instead raw integers
  - Code blocks (e.g. vertex/fragment/etc)
  - In lexer I use numbers to identify different values, but use enums instead so I can use the same ones when parsing AST
  - Generate Shader from parsed AST
  - Make sure includes are handled properly
  - Make sure default values are handled properly
+ - POssible way to handle code blocks: http://flex.sourceforge.net/manual/Start-Conditions.html
+ - When replacing code blocks make sure to replace them with empty lines so that line numbers stay valid when reporting errors
  -----
  - Move all builtin shaders to the new .shader file model
  - Optionally change how GpuPrograms are saved
+ - Save the test shader somewhere so I can use it as a unit test
+  - test shader is missing code blocks
+ - Need a decent way to report syntax errors (at minimum a line and column where it happened)
+  - Later on I'd like a better system that actually tells the user something about the problem, other than just "syntax error"
  -----
  - Add GpuProgram caching (e.g. save compiled programs somewhere for later use)
  - Consider adding restrictions on parameter ranges (probably via an additional qualifier)