WIP modifying lexer so it deals with include files differently

BansheeSL/BansheeSL.vcxproj

@@ -282,6 +282,7 @@
     <ClInclude Include="BsLexerFX.h" />
     <ClInclude Include="BsParserFX.h" />
     <ClInclude Include="Include\BsASTFX.h" />
+    <ClInclude Include="Include\BsIncludeHandler.h" />
     <ClInclude Include="Include\BsSLFXCompiler.h" />
     <ClInclude Include="Include\BsSLImporter.h" />
     <ClInclude Include="Include\BsMMAlloc.h" />
@@ -301,7 +302,8 @@
       <CompileAs Condition="'$(Configuration)|$(Platform)'=='DebugRelease|x64'">Default</CompileAs>
     </ClCompile>
     <ClCompile Include="Source\BsASTFX.c" />
-    <ClCompile Include="Source\BSMMAlloc.c" />
+    <ClCompile Include="Source\BsIncludeHandler.cpp" />
+    <ClCompile Include="Source\BSMMAlloc.cpp" />
     <ClCompile Include="Source\BsSLFXCompiler.cpp" />
     <ClCompile Include="Source\BsSLImporter.cpp" />
     <ClCompile Include="Source\BsSLPlugin.cpp" />

BansheeSL/BansheeSL.vcxproj.filters

@@ -32,6 +32,9 @@
     <ClInclude Include="Include\BsSLFXCompiler.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="Include\BsIncludeHandler.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="BsLexerFX.l" />
@@ -50,14 +53,17 @@
     <ClCompile Include="Source\BsASTFX.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="Source\BSMMAlloc.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="Source\BsSLImporter.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="Source\BsSLFXCompiler.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="Source\BsIncludeHandler.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="Source\BSMMAlloc.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

BansheeSL/BsLexerFX.h

@@ -212,6 +212,8 @@ void *yyalloc (yy_size_t ,yyscan_t yyscanner );
 void *yyrealloc (void *,yy_size_t ,yyscan_t yyscanner );
 void yyfree (void * ,yyscan_t yyscanner );
 
+/* Begin user sect3 */
+
 #define yywrap(yyscanner) 1
 #define YY_SKIP_YYWRAP
 
@@ -219,6 +221,7 @@ void yyfree (void * ,yyscan_t yyscanner );
 
 #ifdef YY_HEADER_EXPORT_START_CONDITIONS
 #define INITIAL 0
+#define INCLUDE 1
 
 #endif
 
@@ -335,8 +338,9 @@ extern int yylex \
 #undef YY_DECL
 #endif
 
-#line 234 "BsLexerFX.l"
+#line 266 "BsLexerFX.l"
+
 
-#line 341 "BsLexerFX.h"
+#line 345 "BsLexerFX.h"
 #undef yyIN_HEADER
 #endif /* yyHEADER_H */

BansheeSL/BsLexerFX.l

@@ -17,6 +17,9 @@ IDENTIFIER		[_a-zA-Z][_a-zA-Z0-9]*
 WS				[ \r\n\t]*
 COMMENT			\/\/[^\n]*
 
+	/* Start conditions */
+%x INCLUDE
+
 %%
 
 {WS}			{ /* Skip blank */ }
@@ -232,3 +235,33 @@ DYNAMIC			{ yylval->intValue = BUV_Dynamic; return TOKEN_BUFFERUSAGE; }
 {COMMENT}		{ }
 {IDENTIFIER}	{ yylval->strValue = mmalloc_strdup(yyextra->memContext, yytext); return TOKEN_IDENTIFIER; }
 .				{ return yytext[0]; }
+
+	/* Preprocessor */
+#include				{ BEGIN(INCLUDE); }
+
+<INCLUDE>{WS}			{ /* Skip blank */ }
+<INCLUDE>{STRING}		{
+	int size = 0;
+	char* includeBuffer = includePush(yyextra, yytext, yylineno, yycolumn, &size);
+	if(!includeBuffer)
+		yyterminate();
+
+	YY_BUFFER_STATE currentBuffer = YY_CURRENT_BUFFER;
+	YY_BUFFER_STATE newBuffer = yy_scan_buffer(includeBuffer, size, yyscanner);
+
+	yy_switch_to_buffer(currentBuffer, yyscanner);
+	yypush_buffer_state(newBuffer, yyscanner);
+
+	BEGIN(INITIAL);
+	}
+<INCLUDE>.				{ return yytext[0]; }
+
+<<EOF>>					{
+	yypop_buffer_state(yyscanner);
+	includePop(yyextra);
+
+	if(!YY_CURRENT_BUFFER)
+		yyterminate();
+}
+
+%%

BansheeSL/BsParserFX.h

@@ -45,6 +45,7 @@ extern int yydebug;
 
 #include "BsMMAlloc.h"
 #include "BsASTFX.h"
+#include "BsIncludeHandler.h"
 
 #ifndef YY_TYPEDEF_YY_SCANNER_T
 #define YY_TYPEDEF_YY_SCANNER_T
@@ -69,7 +70,7 @@ extern int yydebug;
 
 
 /* Line 2579 of glr.c  */
-#line 73 "BsParserFX.h"
+#line 74 "BsParserFX.h"
 
 /* Tokens.  */
 #ifndef YYTOKENTYPE
@@ -198,7 +199,7 @@ extern int yydebug;
 typedef union YYSTYPE
 {
 /* Line 2579 of glr.c  */
-#line 45 "BsParserFX.y"
+#line 46 "BsParserFX.y"
 
 	int intValue;
 	float floatValue;
@@ -210,7 +211,7 @@ typedef union YYSTYPE
 
 
 /* Line 2579 of glr.c  */
-#line 214 "BsParserFX.h"
+#line 215 "BsParserFX.h"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */

BansheeSL/BsParserFX.y

@@ -9,6 +9,7 @@ void yyerror(YYLTYPE *locp, ParseState* parse_state, yyscan_t scanner, const cha
 %code requires{
 #include "BsMMAlloc.h"
 #include "BsASTFX.h"
+#include "BsIncludeHandler.h"
 
 #ifndef YY_TYPEDEF_YY_SCANNER_T
 #define YY_TYPEDEF_YY_SCANNER_T

BansheeSL/Include/BsASTFX.h

@@ -17,6 +17,8 @@ typedef struct tagNodeOptions NodeOptions;
 typedef struct tagNodeOption NodeOption;
 typedef struct tagASTFXNode ASTFXNode;
 typedef struct tagNodeLink NodeLink;
+typedef struct tagIncludeData IncludeData;
+typedef struct tagIncludeLink IncludeLink;
 typedef enum tagFillModeValue FillModeValue;
 typedef enum tagCullModeValue CullModeValue;
 typedef enum tagCompFuncValue CompFuncValue;
@@ -190,6 +192,18 @@ struct tagNodeLink
 	NodeLink* next;
 };
 
+struct tagIncludeData
+{
+	char* filename;
+	char* buffer;
+};
+
+struct tagIncludeLink
+{
+	IncludeData* data;
+	IncludeLink* next;
+};
+
 struct tagParseState
 {
 	ASTFXNode* rootNode;
@@ -202,6 +216,7 @@ struct tagParseState
 	const char* errorMessage;
 
 	NodeLink* nodeStack;
+	IncludeLink* includeStack;
 };
 
 struct tagOptionInfo

BansheeSL/Include/BsIncludeHandler.h

@@ -0,0 +1,19 @@
+//********************************** Banshee Engine (www.banshee3d.com) **************************************************//
+//**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
+#ifndef __INCLUDEHANDLER_H__
+#define __INCLUDEHANDLER_H__
+
+#include "BsASTFX.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	char* includePush(ParseState* state, const char* filename, int line, int column, int* size);
+	void includePop(ParseState* state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

BansheeSL/Include/BsMMAlloc.h

@@ -3,10 +3,18 @@
 #ifndef __MMALLOC_H__
 #define __MMALLOC_H__
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 void* mmalloc_new_context();
 void mmalloc_free_context(void* context);
 void* mmalloc(void* context, int size);
 void mmfree(void* ptr);
 char* mmalloc_strdup(void* context, const char* input);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

BansheeSL/Source/BSMMAlloc.c → BansheeSL/Source/BSMMAlloc.cpp

@@ -64,7 +64,7 @@ void mmfree(void* ptr)
 char* mmalloc_strdup(void* context, const char* input)
 {
 	size_t length = strlen(input);
-	char* output = mmalloc(context, (int)(sizeof(char) * (length + 1)));
+	char* output = (char*)mmalloc(context, (int)(sizeof(char) * (length + 1)));
 
 	memcpy(output, input, length);
 	output[length] = '\0';

BansheeSL/Source/BsASTFX.c

@@ -204,6 +204,7 @@ ParseState* parseStateCreate()
 	parseState->rootNode = nodeCreate(parseState->memContext, NT_Shader);
 	parseState->topNode = 0;
 	parseState->nodeStack = 0;
+	parseState->includeStack = 0;
 
 	parseState->hasError = 0;
 	parseState->errorLine = 0;

BansheeSL/Source/BsIncludeHandler.cpp

@@ -0,0 +1,80 @@
+#include "BsIncludeHandler.h"
+#include "BsSLPrerequisites.h"
+#include "BsShaderManager.h"
+#include "BsShaderInclude.h"
+#include "BsMMAlloc.h"
+
+using namespace BansheeEngine;
+
+char* includePush(ParseState* state, const char* filename, int line, int column, int* size)
+{
+	HShaderInclude include = ShaderManager::instance().findInclude(filename);
+
+	if (include != nullptr)
+		include.blockUntilLoaded();
+
+	int filenameLen = (int)strlen(filename);
+	if (include.isLoaded())
+	{
+		String includeSource = include->getString();
+
+		*size = (int)includeSource.size() + 2;
+		int totalSize = *size + sizeof(IncludeLink) + sizeof(IncludeData) + filenameLen + 1;
+		char* output = (char*)mmalloc(state->memContext, totalSize);
+		char* ptr = output;
+
+		memcpy(ptr, includeSource.data(), *size - 2);
+		ptr[*size - 2] = 0;
+		ptr[*size - 1] = 0;
+
+		ptr += *size;
+		IncludeLink* next = state->includeStack;
+
+		IncludeLink* newLink = (IncludeLink*)ptr;
+		ptr += sizeof(IncludeLink);
+
+		IncludeData* includeData = (IncludeData*)ptr;
+		ptr += sizeof(IncludeData);
+
+		memcpy(ptr, filename, filenameLen);
+		ptr[filenameLen] = '\0';
+
+		includeData->filename = ptr; 
+		includeData->buffer = output;
+
+		newLink->data = includeData;
+		newLink->next = next;
+
+		state->includeStack = newLink;
+		
+		return output;
+	}
+
+	const char* errorLabel = "Error opening include file :";
+	int labelLen = (int)strlen(errorLabel);
+
+	int messageLen = filenameLen + labelLen + 1;
+	char* message = (char*)mmalloc(state->memContext, messageLen);
+
+	memcpy(message, filename, filenameLen);
+	memcpy(message + filenameLen, errorLabel, labelLen);
+	message[messageLen - 1] = '\0';
+
+	state->hasError = 1;
+	state->errorLine = line;
+	state->errorColumn = column;
+	state->errorMessage = message;
+
+	return nullptr;
+}
+
+void includePop(ParseState* state)
+{
+	IncludeLink* current = state->includeStack;
+
+	if (!current)
+		return;
+
+	state->includeStack = current->next;
+	mmfree(current->data->buffer);
+}

BansheeSL/Source/BsSLFXCompiler.cpp

@@ -1553,9 +1553,10 @@ namespace BansheeEngine
 					HShaderInclude include = ShaderManager::instance().findInclude(includePath);
 
 					if (include != nullptr)
-					{
 						include.blockUntilLoaded();
 
+					if (include.isLoaded())
+					{
 						String includeSource = include->getString();
 
 						ParseState* includeParseState = parseStateCreate();