Improved include file and code block parsing for BSL (working)

BansheeSL/BsLexerFX.h

@@ -10,8 +10,6 @@
 
 /* A lexical scanner generated by flex */
 
-/* %not-for-header */
-
 #define FLEX_SCANNER
 #define YY_FLEX_MAJOR_VERSION 2
 #define YY_FLEX_MINOR_VERSION 5
@@ -20,32 +18,16 @@
 #define FLEX_BETA
 #endif
 
-/* %if-c++-only */
-/* %endif */
-
-/* %if-c-only */
-    
-/* %endif */
-
-/* %if-c-only */
-
-/* %endif */
-
 /* First, we deal with  platform-specific or compiler-specific issues. */
 
 /* begin standard C headers. */
-/* %if-c-only */
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
 #include <stdlib.h>
-/* %endif */
 
-/* %if-tables-serialization */
-/* %endif */
 /* end standard C headers. */
 
-/* %if-c-or-c++ */
 /* flex integer type definitions */
 
 #ifndef FLEXINT_H
@@ -110,11 +92,6 @@ typedef unsigned int flex_uint32_t;
 
 #endif /* ! FLEXINT_H */
 
-/* %endif */
-
-/* %if-c++-only */
-/* %endif */
-
 #ifdef __cplusplus
 
 /* The "const" storage-class-modifier is valid. */
@@ -136,12 +113,6 @@ typedef unsigned int flex_uint32_t;
 #define yyconst
 #endif
 
-/* %not-for-header */
-
-/* %not-for-header */
-
-/* %if-reentrant */
-
 /* An opaque pointer. */
 #ifndef YY_TYPEDEF_YY_SCANNER_T
 #define YY_TYPEDEF_YY_SCANNER_T
@@ -159,11 +130,6 @@ typedef void* yyscan_t;
 #define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
 #define yy_flex_debug yyg->yy_flex_debug_r
 
-/* %endif */
-
-/* %if-not-reentrant */
-/* %endif */
-
 /* Size of default input buffer. */
 #ifndef YY_BUF_SIZE
 #define YY_BUF_SIZE 16384
@@ -179,24 +145,11 @@ typedef struct yy_buffer_state *YY_BUFFER_STATE;
 typedef size_t yy_size_t;
 #endif
 
-/* %if-not-reentrant */
-/* %endif */
-
-/* %if-c-only */
-/* %if-not-reentrant */
-/* %endif */
-/* %endif */
-
 #ifndef YY_STRUCT_YY_BUFFER_STATE
 #define YY_STRUCT_YY_BUFFER_STATE
 struct yy_buffer_state
 	{
-/* %if-c-only */
 	FILE *yy_input_file;
-/* %endif */
-
-/* %if-c++-only */
-/* %endif */
 
 	char *yy_ch_buf;		/* input buffer */
 	char *yy_buf_pos;		/* current position in input buffer */
@@ -243,18 +196,6 @@ struct yy_buffer_state
 	};
 #endif /* !YY_STRUCT_YY_BUFFER_STATE */
 
-/* %if-c-only Standard (non-C++) definition */
-/* %not-for-header */
-
-/* %endif */
-
-/* %if-c-only Standard (non-C++) definition */
-
-/* %if-not-reentrant */
-/* %not-for-header */
-
-/* %endif */
-
 void yyrestart (FILE *input_file ,yyscan_t yyscanner );
 void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
 YY_BUFFER_STATE yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner );
@@ -267,29 +208,24 @@ YY_BUFFER_STATE yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner );
 YY_BUFFER_STATE yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner );
 YY_BUFFER_STATE yy_scan_bytes (yyconst char *bytes,yy_size_t len ,yyscan_t yyscanner );
 
-/* %endif */
-
 void *yyalloc (yy_size_t ,yyscan_t yyscanner );
 void *yyrealloc (void *,yy_size_t ,yyscan_t yyscanner );
 void yyfree (void * ,yyscan_t yyscanner );
 
-/* %% [1.0] yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here */
 /* Begin user sect3 */
 
 #define yywrap(yyscanner) 1
 #define YY_SKIP_YYWRAP
 
-#define FLEX_DEBUG
-
 #define yytext_ptr yytext_r
 
-/* %if-c-only Standard (non-C++) definition */
-
-/* %endif */
-
 #ifdef YY_HEADER_EXPORT_START_CONDITIONS
 #define INITIAL 0
 #define INCLUDE 1
+#define CODEBLOCK_HEADER 2
+#define CODEBLOCK_EQUALS 3
+#define CODEBLOCK 4
+#define CODEBLOCK_END 5
 
 #endif
 
@@ -300,23 +236,10 @@ void yyfree (void * ,yyscan_t yyscanner );
     
 #define YY_EXTRA_TYPE struct tagParseState *
 
-/* %if-c-only Reentrant structure and macros (non-C++). */
-/* %if-reentrant */
-
-/* %if-c-only */
-
-/* %endif */
-
-/* %if-reentrant */
-
 int yylex_init (yyscan_t* scanner);
 
 int yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner);
 
-/* %endif */
-
-/* %endif End reentrant structures and macros. */
-
 /* Accessor methods to globals.
    These are made visible to non-reentrant scanners for convenience. */
 
@@ -350,8 +273,6 @@ int yyget_column  (yyscan_t yyscanner );
 
 void yyset_column (int column_no ,yyscan_t yyscanner );
 
-/* %if-bison-bridge */
-
 YYSTYPE * yyget_lval (yyscan_t yyscanner );
 
 void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
@@ -360,8 +281,6 @@ void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
     
         void yyset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner );
     
-/* %endif */
-
 /* Macros after this point can all be overridden by user definitions in
  * section 1.
  */
@@ -374,10 +293,6 @@ extern int yywrap (yyscan_t yyscanner );
 #endif
 #endif
 
-/* %not-for-header */
-
-/* %endif */
-
 #ifndef yytext_ptr
 static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner);
 #endif
@@ -387,16 +302,9 @@ static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
 #endif
 
 #ifndef YY_NO_INPUT
-/* %if-c-only Standard (non-C++) definition */
-/* %not-for-header */
 
-/* %endif */
 #endif
 
-/* %if-c-only */
-
-/* %endif */
-
 /* Amount of stuff to slurp up with each read. */
 #ifndef YY_READ_BUF_SIZE
 #define YY_READ_BUF_SIZE 8192
@@ -407,40 +315,21 @@ static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
 #define YY_START_STACK_INCR 25
 #endif
 
-/* %if-tables-serialization structures and prototypes */
-/* %not-for-header */
-
-/* %not-for-header */
-
 /* Default declaration of generated scanner - a define so the user can
  * easily add parameters.
  */
 #ifndef YY_DECL
 #define YY_DECL_IS_OURS 1
-/* %if-c-only Standard (non-C++) definition */
 
 extern int yylex \
                (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner);
 
 #define YY_DECL int yylex \
                (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner)
-/* %endif */
-/* %if-c++-only C++ definition */
-/* %endif */
 #endif /* !YY_DECL */
 
-/* %not-for-header */
-
-/* %if-c++-only */
-/* %not-for-header */
-
-/* %endif */
-
 /* yy_get_previous_state - get the state just before the EOB char was reached */
 
-/* %if-c-only */
-/* %not-for-header */
-
 #undef YY_NEW_FILE
 #undef YY_FLUSH_BUFFER
 #undef yy_set_bol
@@ -453,9 +342,9 @@ extern int yylex \
 #undef YY_DECL
 #endif
 
-#line 265 "BsLexerFX.l"
+#line 308 "BsLexerFX.l"
 
 
-#line 460 "BsLexerFX.h"
+#line 349 "BsLexerFX.h"
 #undef yyIN_HEADER
 #endif /* yyHEADER_H */

BansheeSL/BsLexerFX.l

@@ -5,7 +5,7 @@
 #define YY_USER_INIT yylineno = 0; yycolumn = 0;
 %}
  
-%option yylineno reentrant noyywrap nounistd never-interactive warn nodefault bison-bridge bison-locations debug
+%option yylineno reentrant noyywrap nounistd never-interactive warn nodefault bison-bridge bison-locations
 %option outfile="BsLexerFX.c" header-file="BsLexerFX.h"
 %option extra-type="struct tagParseState *"
 
@@ -15,10 +15,15 @@ FLOAT			[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?[fF]?
 STRING			\"[^"\n]*\"
 IDENTIFIER		[_a-zA-Z][_a-zA-Z0-9]*
 WS				[ \r\n\t]*
+SINGLEWS		[ \r\n\t]
 COMMENT			\/\/[^\n]*
 
 	/* Start conditions */
 %x INCLUDE
+%x CODEBLOCK_HEADER
+%x CODEBLOCK_EQUALS
+%x CODEBLOCK
+%x CODEBLOCK_END
 
 %%
 
@@ -93,13 +98,6 @@ Language		{ return TOKEN_LANGUAGE; }
 Pass			{ return TOKEN_PASS; }
 
 	/* Pass keywords */
-Vertex			{ return TOKEN_VERTEX; }
-Fragment		{ return TOKEN_FRAGMENT; }
-Geometry		{ return TOKEN_GEOMETRY; }
-Hull			{ return TOKEN_HULL; }
-Domain			{ return TOKEN_DOMAIN; }
-Compute			{ return TOKEN_COMPUTE; }
-Common			{ return TOKEN_COMMON; }
 StencilRef		{ return TOKEN_STENCILREF; }
 
 	/* Rasterizer state keywords */
@@ -231,10 +229,6 @@ ANISOC			{ yylval->intValue = FV_AnisotropicCmp; return TOKEN_FILTERVALUE; }
 STATIC			{ yylval->intValue = BUV_Static; return TOKEN_BUFFERUSAGE; }
 DYNAMIC			{ yylval->intValue = BUV_Dynamic; return TOKEN_BUFFERUSAGE; }
 
-{COMMENT}		{ }
-{IDENTIFIER}	{ yylval->strValue = mmalloc_strdup(yyextra->memContext, yytext); return TOKEN_IDENTIFIER; }
-.				{ return yytext[0]; }
-
 	/* Preprocessor */
 #include				{ BEGIN(INCLUDE); }
 
@@ -263,4 +257,53 @@ DYNAMIC			{ yylval->intValue = BUV_Dynamic; return TOKEN_BUFFERUSAGE; }
 	includePop(yyextra);
 }
 
+	/* Code blocks */
+Vertex			{ BEGIN(CODEBLOCK_HEADER); return TOKEN_VERTEX; }
+Fragment		{ BEGIN(CODEBLOCK_HEADER); return TOKEN_FRAGMENT; }
+Geometry		{ BEGIN(CODEBLOCK_HEADER); return TOKEN_GEOMETRY; }
+Hull			{ BEGIN(CODEBLOCK_HEADER); return TOKEN_HULL; }
+Domain			{ BEGIN(CODEBLOCK_HEADER); return TOKEN_DOMAIN; }
+Compute			{ BEGIN(CODEBLOCK_HEADER); return TOKEN_COMPUTE; }
+Common			{ BEGIN(CODEBLOCK_HEADER); return TOKEN_COMMON; }
+
+	/* Track when the code block begins, insert all code block characters into our own buffer, record a sequential index */
+	/* of all code blocks in the text, and track bracket open/closed state so we know when we're done with the code block. */
+	/* And finally output a sequential code block index to the parser (it shouldn't be aware of anything else in the block). */
+<CODEBLOCK_HEADER>=		{ BEGIN(CODEBLOCK_EQUALS); return yytext[0]; }
+<CODEBLOCK_HEADER>{WS}	{ /* Skip blank */ }
+<CODEBLOCK_HEADER>.		{ return yytext[0]; }
+
+<CODEBLOCK_EQUALS>\{	{ BEGIN(CODEBLOCK); beginCodeBlock(yyextra); yyextra->numOpenBrackets = 1; return yytext[0]; }
+<CODEBLOCK_EQUALS>{WS}	{ /* Skip blank */ }
+<CODEBLOCK_EQUALS>.		{ return yytext[0]; }
+
+<CODEBLOCK>\{			{ yyextra->numOpenBrackets++; appendCodeBlock(yyextra, yytext[0]); }
+<CODEBLOCK>\}			{ 
+	yyextra->numOpenBrackets--; 
+
+	if(yyextra->numOpenBrackets == 0)
+	{
+		BEGIN(CODEBLOCK_END);
+		unput('0');
+	}
+	else
+		appendCodeBlock(yyextra, yytext[0]);
+}
+<CODEBLOCK>.|{SINGLEWS}		{ appendCodeBlock(yyextra, yytext[0]); }
+
+	/* Logic for manually inserting "Index = codeBlockIndex;". We insert arbitrary numbers which allows us to sequentially */
+	/* output all the tokens we need. We use only single-character values so we don't override anything in the text buffer */
+	/* (since the starting value was also a single character "{"). */
+<CODEBLOCK_END>0	{ unput('1'); return TOKEN_INDEX; }
+<CODEBLOCK_END>1	{ unput('2'); return '='; }
+<CODEBLOCK_END>2	{ yylval->intValue = getCodeBlockIndex(yyextra); unput('3'); return TOKEN_INTEGER; }
+<CODEBLOCK_END>3	{ unput('4'); return ';'; }
+<CODEBLOCK_END>4	{ BEGIN(INITIAL); return '}'; }
+<CODEBLOCK_END>.|{WS}	{ /* Never reached */ }
+
+	/* Catch all rules */
+{COMMENT}			{ }
+{IDENTIFIER}		{ yylval->strValue = mmalloc_strdup(yyextra->memContext, yytext); return TOKEN_IDENTIFIER; }
+.					{ return yytext[0]; }
+
 %%

BansheeSL/BsParserFX.h

@@ -34,7 +34,7 @@
 # define YY_YY_BSPARSERFX_H_INCLUDED
 /* Enabling traces.  */
 #ifndef YYDEBUG
-# define YYDEBUG 1
+# define YYDEBUG 0
 #endif
 #if YYDEBUG
 extern int yydebug;
@@ -198,7 +198,7 @@ extern int yydebug;
 typedef union YYSTYPE
 {
 /* Line 2579 of glr.c  */
-#line 47 "BsParserFX.y"
+#line 46 "BsParserFX.y"
 
 	int intValue;
 	float floatValue;

BansheeSL/BsParserFX.y

@@ -42,7 +42,6 @@ void yyerror(YYLTYPE *locp, ParseState* parse_state, yyscan_t scanner, const cha
 %parse-param { ParseState* parse_state }
 %parse-param { yyscan_t scanner }
 %glr-parser
-%debug
 
 %union {
 	int intValue;
@@ -366,37 +365,37 @@ code
 code_header
 	: TOKEN_VERTEX '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeVertex); 
 			nodePush(parse_state, $$);
 		}
 	| TOKEN_FRAGMENT '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeFragment); 
 			nodePush(parse_state, $$);
 		}
 	| TOKEN_GEOMETRY '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeGeometry); 
 			nodePush(parse_state, $$);
 		}
 	| TOKEN_HULL '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeHull); 
 			nodePush(parse_state, $$);
 		}
 	| TOKEN_DOMAIN '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeDomain); 
 			nodePush(parse_state, $$);
 		}
 	| TOKEN_COMPUTE '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeCompute); 
 			nodePush(parse_state, $$);
 		}
 	| TOKEN_COMMON '='
 		{ 
-			$$ = nodeCreate(parse_state->memContext, NT_Code); 
+			$$ = nodeCreate(parse_state->memContext, NT_CodeCommon); 
 			nodePush(parse_state, $$);
 		}
 	;

BansheeSL/Include/BsASTFX.h

@@ -43,7 +43,13 @@ enum tagNodeType
 	NT_AddrMode,
 	NT_Parameter,
 	NT_Block,
-	NT_Code
+	NT_CodeVertex,
+	NT_CodeFragment,
+	NT_CodeGeometry,
+	NT_CodeHull,
+	NT_CodeDomain,
+	NT_CodeCompute,
+	NT_CodeCommon,
 };
 
 enum tagOptionType
@@ -207,8 +213,9 @@ struct tagIncludeLink
 struct tagCodeString
 {
 	char* code;
-	int type;
 	int index;
+	int size;
+	int capacity;
 
 	CodeString* next;
 };
@@ -229,6 +236,7 @@ struct tagParseState
 	IncludeLink* includes;
 	CodeString* codeStrings;
 	int numCodeStrings;
+	int numOpenBrackets;
 };
 
 struct tagOptionInfo
@@ -282,7 +290,9 @@ void nodeDelete(ASTFXNode* node);
 void nodePush(ParseState* parseState, ASTFXNode* node);
 void nodePop(ParseState* parseState);
 
-void addCodeBlock(ParseState* parseState, int type, char* code, int codeLength);
+void beginCodeBlock(ParseState* parseState);
+void appendCodeBlock(ParseState* parseState, char value);
+int getCodeBlockIndex(ParseState* parseState);
 
 ParseState* parseStateCreate();
 void parseStateDelete(ParseState* parseState);

BansheeSL/Include/BsSLFXCompiler.h

@@ -33,15 +33,6 @@ namespace BansheeEngine
 			Vertex, Fragment, Geometry, Hull, Domain, Compute, Common
 		};
 
-		/**
-		 * Represents a block of code written in a GPU program language for a specific GPU program type. (i.e. non-FX code)
-		 */
-		struct CodeBlock
-		{
-			CodeBlockType type;
-			String code;
-		};
-
 		/**	Temporary data describing a pass during parsing. */
 		struct PassData
 		{
@@ -71,6 +62,7 @@ namespace BansheeEngine
 			StringID renderAPI = RenderAPIAny;
 			String language;
 
+			PassData commonPassData;
 			Vector<PassData> passes;
 		};
 
@@ -82,13 +74,6 @@ namespace BansheeEngine
 		/** Converts the provided source into an abstract syntax tree using the lexer & parser for BSL FX syntax. */
 		static void parseFX(ParseState* parseState, const char* source);
 
-		/**
-		 * Retrieves non-FX code blocks (i.e. actual shader code) from the source code and removes them from the input so 
-		 * all that remains is a pure FX code. Found blocks are returned so they may be compiled using their appropriate 
-		 * compiler.
-		 */
-		static Vector<CodeBlock> parseCodeBlocks(String& source);
-
 		/**
 		 * Retrieves the renderer and language specified for the technique. These two values are considered a unique 
 		 * identifier for a technique.
@@ -199,30 +184,30 @@ namespace BansheeEngine
 		 * Parses a code AST node and outputs the result in one of the streams within the provided pass data.
 		 *
 		 * @param[in]	codeNode	AST node to parse
-		 * @param[in]	codeBlocks	GPU program source code retrieved from parseCodeBlocks().
+		 * @param[in]	codeBlocks	GPU program source code.
 		 * @param[in]	passData	Pass data containing temporary pass data, including the code streams that the code 
 		 *							block code will be written to.
 		 */
-		static void parseCodeBlock(ASTFXNode* codeNode, const Vector<CodeBlock>& codeBlocks, PassData& passData);
+		static void parseCodeBlock(ASTFXNode* codeNode, const Vector<String>& codeBlocks, PassData& passData);
 
 		/**
 		 * Parses the pass AST node and populates the provided @passData with all relevant pass parameters.
 		 *
 		 * @param[in]	passNode		Node to parse.
-		 * @param[in]	codeBlocks		GPU program source code retrieved from parseCodeBlocks().
+		 * @param[in]	codeBlocks		GPU program source code.
 		 * @param[out]	passData		Will contain pass data after parsing. 
 		 */
-		static void parsePass(ASTFXNode* passNode, const Vector<CodeBlock>& codeBlocks, PassData& passData);
+		static void parsePass(ASTFXNode* passNode, const Vector<String>& codeBlocks, PassData& passData);
 
 		/**
 		 * Parses the technique AST node and generates a single technique object. Returns null if no technique can be 
 		 * parsed.
 		 *
 		 * @param[in]	techniqueNode	Node to parse.
-		 * @param[in]	codeBlocks		GPU program source code retrieved from parseCodeBlocks().
+		 * @param[in]	codeBlocks		GPU program source code.
 		 * @param[out]	techniqueData	Will contain technique data after parsing.
 		 */
-		static void parseTechnique(ASTFXNode* techniqueNode, const Vector<CodeBlock>& codeBlocks, TechniqueData& techniqueData);
+		static void parseTechnique(ASTFXNode* techniqueNode, const Vector<String>& codeBlocks, TechniqueData& techniqueData);
 
 		/**
 		 * Parses the parameters AST node and populates the shader descriptor with information about GPU program parameters
@@ -242,10 +227,10 @@ namespace BansheeEngine
 		 * @param[in]		name		Optional name for the shader.
 		 * @param[in, out]	parseState	Parser state object that has previously been initialized with the AST using 
 		 *								parseFX().
-		 * @param	codeBlocks			GPU program source code retrieved from parseCodeBlocks().
+		 * @param	codeBlocks			GPU program source code.
 		 * @return						A result object containing the shader if successful, or error message if not.
 		 */
-		static BSLFXCompileResult parseShader(const String& name, ParseState* parseState, Vector<CodeBlock>& codeBlocks);
+		static BSLFXCompileResult parseShader(const String& name, ParseState* parseState, Vector<String>& codeBlocks);
 
 		/**
 		 * Converts a null-terminated string into a standard string, and eliminates quotes that are assumed to be at the 

BansheeSL/Source/BsASTFX.c

@@ -196,22 +196,42 @@ void nodePop(ParseState* parseState)
 	mmfree(toRemove);
 }
 
-void addCodeBlock(ParseState* parseState, int type, char* code, int codeLength)
+void beginCodeBlock(ParseState* parseState)
 {
-	char* buffer = mmalloc(parseState->memContext, sizeof(CodeString) + codeLength);
-	
-	CodeString* codeString = (CodeString*)buffer;
-	codeString->type = type;
+	CodeString* codeString = (CodeString*)mmalloc(parseState->memContext, sizeof(CodeString));
 	codeString->index = parseState->numCodeStrings;
-	codeString->code = buffer + sizeof(CodeString);
+	codeString->size = 0;
+	codeString->capacity = 128;
+	codeString->code = mmalloc(parseState->memContext, codeString->capacity);
 	codeString->next = parseState->codeStrings;
 
-	memcpy(codeString->code, code, codeLength);
-
 	parseState->numCodeStrings++;
 	parseState->codeStrings = codeString;
 }
 
+void appendCodeBlock(ParseState* parseState, char value)
+{
+	CodeString* codeString = parseState->codeStrings;
+
+	if((codeString->size + 1) > codeString->capacity)
+	{
+		int newCapacity = codeString->capacity * 2;
+		char* newBuffer = mmalloc(parseState->memContext, newCapacity);
+		memcpy(newBuffer, codeString->code, codeString->size);
+		mmfree(codeString->code);
+
+		codeString->code = newBuffer;
+		codeString->capacity = newCapacity;
+	}
+
+	codeString->code[codeString->size++] = value;
+}
+
+int getCodeBlockIndex(ParseState* parseState)
+{
+	return parseState->codeStrings->index;
+}
+
 ParseState* parseStateCreate()
 {
 	ParseState* parseState = (ParseState*)malloc(sizeof(ParseState));
@@ -223,6 +243,7 @@ ParseState* parseStateCreate()
 	parseState->includes = 0;
 	parseState->codeStrings = 0;
 	parseState->numCodeStrings = 0;
+	parseState->numOpenBrackets = 0;
 
 	parseState->hasError = 0;
 	parseState->errorLine = 0;

BansheeSL/Source/BsSLFXCompiler.cpp

@@ -73,7 +73,6 @@ namespace BansheeEngine
 		String parsedSource = source;
 
 		ParseState* parseState = parseStateCreate();
-		Vector<CodeBlock> codeBlocks = parseCodeBlocks(parsedSource);
 		parseFX(parseState, parsedSource.c_str());
 
 		if (parseState->hasError > 0)
@@ -89,6 +88,17 @@ namespace BansheeEngine
 			// Only enable for debug purposes
 			//SLFXDebugPrint(parseState->rootNode, "");
 
+			Vector<String> codeBlocks;
+			CodeString* codeString = parseState->codeStrings;
+			while(codeString != nullptr)
+			{
+				while ((INT32)codeBlocks.size() <= codeString->index)
+					codeBlocks.push_back(String());
+
+				codeBlocks[codeString->index] = String(codeString->code, codeString->size);
+				codeString = codeString->next;
+			}
+
 			output = parseShader("Shader", parseState, codeBlocks);
 
 			StringStream gpuProgError;
@@ -150,10 +160,10 @@ namespace BansheeEngine
 			return;
 
 		// If debug output from lexer is needed uncomment this and add %debug option to lexer file
-		yyset_debug(true, scanner);
+		//yyset_debug(true, scanner);
 
 		// If debug output from parser is needed uncomment this and add %debug option to parser file
-		yydebug = true;
+		//yydebug = true;
 
 		state = yy_scan_string(source, scanner);
 
@@ -164,73 +174,6 @@ namespace BansheeEngine
 		yylex_destroy(scanner);
 	}
 
-	Vector<BSLFXCompiler::CodeBlock> BSLFXCompiler::parseCodeBlocks(String& source)
-	{
-		std::regex pattern = std::regex(R"((Vertex|Fragment|Geometry|Hull|Domain|Compute|Common)\s*=\s*\{)");
-		std::smatch matches;
-
-		Vector<CodeBlock> codeBlocks;
-
-		UINT32 offset = 0;
-		while (std::regex_search(source.cbegin() + offset, source.cend(), matches, pattern))
-		{
-			UINT32 idx = (UINT32)codeBlocks.size();
-
-			codeBlocks.push_back(CodeBlock());
-			CodeBlock& newBlock = codeBlocks.back();
-
-			std::string type = matches[1].str();
-			if (type == "Vertex")
-				newBlock.type = CodeBlockType::Vertex;
-			else if (type == "Fragment")
-				newBlock.type = CodeBlockType::Fragment;
-			else if (type == "Geometry")
-				newBlock.type = CodeBlockType::Geometry;
-			else if (type == "Hull")
-				newBlock.type = CodeBlockType::Hull;
-			else if (type == "Domain")
-				newBlock.type = CodeBlockType::Domain;
-			else if (type == "Compute")
-				newBlock.type = CodeBlockType::Compute;
-			else if (type == "Common")
-				newBlock.type = CodeBlockType::Common;
-
-			offset += (UINT32)matches.position() + (UINT32)matches.length();
-
-			StringStream newDataStream;
-			newDataStream << "Index = " + toString(idx) + ";";
-
-			StringStream codeStream;
-			UINT32 ummatchedBrackets = 1;
-			for (UINT32 i = offset; i < (UINT32)source.length(); i++)
-			{
-				if (source[i] == '{')
-					ummatchedBrackets++;
-
-				if (source[i] == '}')
-					ummatchedBrackets--;
-
-				if (ummatchedBrackets == 0)
-					break;
-
-				if (source[i] == '\r' || source[i] == '\n')
-					newDataStream << source[i];
-
-				codeStream << source[i];
-			}
-
-			newBlock.code = codeStream.str();
-
-			source.erase(source.cbegin() + offset, source.cbegin() + offset + (UINT32)newBlock.code.size());
-
-			String newData = newDataStream.str();
-			source.insert(offset, newData);
-			offset += (UINT32)newData.size();
-		}
-
-		return codeBlocks;
-	}
-
 	void BSLFXCompiler::getTechniqueIdentifier(ASTFXNode* technique, StringID& renderer, String& language)
 	{
 		renderer = RendererAny;
@@ -897,10 +840,14 @@ namespace BansheeEngine
 		return SamplerState::create(desc);
 	}
 
-	void BSLFXCompiler::parseCodeBlock(ASTFXNode* codeNode, const Vector<CodeBlock>& codeBlocks, PassData& passData)
+	void BSLFXCompiler::parseCodeBlock(ASTFXNode* codeNode, const Vector<String>& codeBlocks, PassData& passData)
 	{
-		if (codeNode == nullptr || codeNode->type != NT_Code)
+		if (codeNode == nullptr || (codeNode->type != NT_CodeCommon && codeNode->type != NT_CodeVertex && 
+			codeNode->type != NT_CodeFragment && codeNode->type != NT_CodeGeometry && codeNode->type != NT_CodeHull &&
+			codeNode->type != NT_CodeDomain && codeNode->type != NT_CodeCompute))
+		{
 			return;
+		}
 
 		UINT32 index = (UINT32)-1;
 		for (int j = 0; j < codeNode->options->count; j++)
@@ -911,35 +858,34 @@ namespace BansheeEngine
 
 		if (index != (UINT32)-1 && index < (UINT32)codeBlocks.size())
 		{
-			const CodeBlock& codeBlock = codeBlocks[index];
-			switch (codeBlock.type)
+			switch (codeNode->type)
 			{
-			case CodeBlockType::Vertex:
-				passData.vertexCode += codeBlock.code;
+			case NT_CodeVertex:
+				passData.vertexCode += codeBlocks[index];
 				break;
-			case CodeBlockType::Fragment:
-				passData.fragmentCode += codeBlock.code;
+			case NT_CodeFragment:
+				passData.fragmentCode += codeBlocks[index];
 				break;
-			case CodeBlockType::Geometry:
-				passData.geometryCode += codeBlock.code;
+			case NT_CodeGeometry:
+				passData.geometryCode += codeBlocks[index];
 				break;
-			case CodeBlockType::Hull:
-				passData.hullCode += codeBlock.code;
+			case NT_CodeHull:
+				passData.hullCode += codeBlocks[index];
 				break;
-			case CodeBlockType::Domain:
-				passData.domainCode += codeBlock.code;
+			case NT_CodeDomain:
+				passData.domainCode += codeBlocks[index];
 				break;
-			case CodeBlockType::Compute:
-				passData.computeCode += codeBlock.code;
+			case NT_CodeCompute:
+				passData.computeCode += codeBlocks[index];
 				break;
-			case CodeBlockType::Common:
-				passData.commonCode += codeBlock.code;
+			case NT_CodeCommon:
+				passData.commonCode += codeBlocks[index];
 				break;
 			}
 		}
 	}
 
-	void BSLFXCompiler::parsePass(ASTFXNode* passNode, const Vector<CodeBlock>& codeBlocks, PassData& passData)
+	void BSLFXCompiler::parsePass(ASTFXNode* passNode, const Vector<String>& codeBlocks, PassData& passData)
 	{
 		if (passNode == nullptr || passNode->type != NT_Pass)
 			return;
@@ -964,14 +910,14 @@ namespace BansheeEngine
 		}
 	}
 
-	void BSLFXCompiler::parseTechnique(ASTFXNode* techniqueNode, const Vector<CodeBlock>& codeBlocks, TechniqueData& techniqueData)
+	void BSLFXCompiler::parseTechnique(ASTFXNode* techniqueNode, const Vector<String>& codeBlocks, TechniqueData& techniqueData)
 	{
 		if (techniqueNode == nullptr || techniqueNode->type != NT_Technique)
 			return;
 
-		PassData commonPassData;
 		UINT32 nextPassIdx = 0;
-		for (int i = 0; i < techniqueNode->options->count; i++)
+		// Go in reverse because options are added in reverse order during parsing
+		for (int i = techniqueNode->options->count - 1; i >= 0; i--)
 		{
 			NodeOption* option = &techniqueNode->options->entries[i];
 
@@ -1015,12 +961,12 @@ namespace BansheeEngine
 				passData->rasterizerIsDefault &= !parseRasterizerState(passData->rasterizerDesc, techniqueNode);
 				passData->depthStencilIsDefault &= !parseDepthStencilState(passData->depthStencilDesc, techniqueNode);
 
-				passData->vertexCode = commonPassData.vertexCode + passData->vertexCode;
-				passData->fragmentCode = commonPassData.fragmentCode + passData->fragmentCode;
-				passData->geometryCode = commonPassData.geometryCode + passData->geometryCode;
-				passData->hullCode = commonPassData.hullCode + passData->hullCode;
-				passData->domainCode = commonPassData.domainCode + passData->domainCode;
-				passData->commonCode = commonPassData.commonCode + passData->commonCode;
+				passData->vertexCode = techniqueData.commonPassData.vertexCode + passData->vertexCode;
+				passData->fragmentCode = techniqueData.commonPassData.fragmentCode + passData->fragmentCode;
+				passData->geometryCode = techniqueData.commonPassData.geometryCode + passData->geometryCode;
+				passData->hullCode = techniqueData.commonPassData.hullCode + passData->hullCode;
+				passData->domainCode = techniqueData.commonPassData.domainCode + passData->domainCode;
+				passData->commonCode = techniqueData.commonPassData.commonCode + passData->commonCode;
 				
 				parsePass(passNode, codeBlocks, *passData);
 			}
@@ -1032,7 +978,7 @@ namespace BansheeEngine
 				parseLanguage(removeQuotes(option->value.strValue), techniqueData.renderAPI, techniqueData.language);
 				break;
 			case OT_Code:
-				parseCodeBlock(option->value.nodePtr, codeBlocks, commonPassData);
+				parseCodeBlock(option->value.nodePtr, codeBlocks, techniqueData.commonPassData);
 				break;
 			}
 		}
@@ -1186,7 +1132,7 @@ namespace BansheeEngine
 		}
 	}
 
-	BSLFXCompileResult BSLFXCompiler::parseShader(const String& name, ParseState* parseState, Vector<CodeBlock>& codeBlocks)
+	BSLFXCompileResult BSLFXCompiler::parseShader(const String& name, ParseState* parseState, Vector<String>& codeBlocks)
 	{
 		BSLFXCompileResult output;
 
@@ -1199,7 +1145,8 @@ namespace BansheeEngine
 		SHADER_DESC shaderDesc;
 		Vector<pair<ASTFXNode*, TechniqueData>> techniqueData;
 
-		for (int i = 0; i < parseState->rootNode->options->count; i++)
+		// Go in reverse because options are added in reverse order during parsing
+		for (int i = parseState->rootNode->options->count - 1; i >= 0; i--)
 		{
 			NodeOption* option = &parseState->rootNode->options->entries[i];