- Scanner refactoring

- Exception refactoring

anki/core/AsyncLoader.cpp

@@ -64,7 +64,8 @@ void AsyncLoader::workingFunc()
 		}
 		catch(std::exception& e)
 		{
-			ANKI_ERROR("Loading \"" << req.filename << "\" failed: " << e.what());
+			ANKI_ERROR("Loading \"" << req.filename <<
+				"\" failed: " << e.what());
 			ok = false;
 		}
 

anki/misc/Parser.h

@@ -3,7 +3,7 @@
 
 #include <boost/lexical_cast.hpp>
 #include "anki/util/Exception.h"
-#include "anki/util/scanner/Scanner.h"
+#include "anki/util/Scanner.h"
 
 
 namespace anki {

anki/misc/Parser.inl.h

@@ -132,7 +132,7 @@ void parseNumber(scanner::Scanner& scanner, bool sign, Type& out)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION(e.what());
+		throw ANKI_EXCEPTION_R("Error", e);
 	}
 }
 
@@ -171,7 +171,7 @@ void parseMathVector(scanner::Scanner& scanner, Type& out)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION(e.what());
+		throw ANKI_EXCEPTION_R("Error", e);
 	}
 }
 

anki/renderer/Bl.cpp

@@ -43,8 +43,8 @@ void Bl::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create horizontal blur "
-			"post-processing stage FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create horizontal blur "
+			"post-processing stage FBO: ", e);
 	}
 
 	hBlurSProg.load(ShaderProgram::createSrcCodeToCache(
@@ -61,8 +61,8 @@ void Bl::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create vertical blur "
-			"post-processing stage FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create vertical blur "
+			"post-processing stage FBO", e);
 	}
 
 	vBlurSProg.load(ShaderProgram::createSrcCodeToCache(
@@ -79,9 +79,8 @@ void Bl::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create side blur "
-			"post-processing stage FBO: " +
-			e.what());
+		throw ANKI_EXCEPTION_R("Cannot create side blur "
+			"post-processing stage FBO", e);
 	}
 
 	sideBlurMap.load("engine-rsrc/side-blur.png");

anki/renderer/Dbg.cpp

@@ -226,7 +226,7 @@ void Dbg::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create debug FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create debug FBO", e);
 	}
 
 	//

anki/renderer/Hdr.cpp

@@ -54,8 +54,8 @@ void Hdr::initFbo(Fbo& fbo, Texture& fai)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create deferred shading post-processing "
-			"stage HDR passes FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create deferred shading post-processing "
+			"stage HDR passes FBO", e);
 	}
 }
 

anki/renderer/Is.cpp

@@ -64,8 +64,8 @@ void Is::initFbo()
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create deferred shading illumination "
-			"stage FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create deferred shading illumination "
+			"stage FBO", e);
 	}
 }
 
@@ -98,8 +98,8 @@ void Is::initCopy()
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create deferred shading "
-			"illumination stage additional FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create deferred shading "
+			"illumination stage additional FBO", e);
 	}
 }
 

anki/renderer/MainRenderer.cpp

@@ -109,7 +109,7 @@ void MainRenderer::initGl()
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("OpenGL initialization failed: " + e.what());
+		throw ANKI_EXCEPTION_R("OpenGL initialization failed", e);
 	}
 }
 

anki/renderer/Ms.cpp

@@ -74,8 +74,8 @@ void Ms::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create deferred shading material stage FBO: " +
-			e.what());
+		throw ANKI_EXCEPTION_R("Cannot create deferred "
+			"shading material stage FBO", e);
 	}
 
 	ez.init(initializer);

anki/renderer/Pps.cpp

@@ -53,8 +53,8 @@ void Pps::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create pre-pass post-processing stage FBO: " +
-			e.what());
+		throw ANKI_EXCEPTION_R("Cannot create pre-pass "
+			"post-processing stage FBO", e);
 	}
 
 	// SProg
@@ -86,8 +86,8 @@ void Pps::init(const RendererInitializer& initializer)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create post-pass post-processing stage FBO: " +
-			e.what());
+		throw ANKI_EXCEPTION_R("Cannot create post-pass "
+			"post-processing stage FBO", e);
 	}
 
 	// SProg

anki/renderer/Sm.cpp

@@ -94,7 +94,8 @@ void Sm::initLevel(uint resolution, float distance, bool bilinear, Level& level)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create shadowmapping FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create shadowmapping "
+			"FBO", e);
 	}
 }
 

anki/renderer/Ssao.cpp

@@ -41,8 +41,8 @@ void Ssao::createFbo(Fbo& fbo, Texture& fai)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot create deferred shading post-processing "
-			"stage SSAO blur FBO: " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot create deferred shading post-processing "
+			"stage SSAO blur FBO", e);
 	}
 }
 

anki/resource/Image.cpp

@@ -455,7 +455,7 @@ void Image::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("File \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("File \"" + filename + "\": ", e);
 	}
 }
 

anki/resource/LightRsrc.cpp

@@ -216,7 +216,7 @@ void LightRsrc::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Material \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Material \"" + filename + "\": ", e);
 	}
 }
 

anki/resource/Material.cpp

@@ -65,7 +65,7 @@ void Material::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("File \"" + filename + "\" failed: " + e.what());
+		throw ANKI_EXCEPTION_R("File \"" + filename + "\" failed", e);
 	}
 }
 

anki/resource/Mesh.cpp

@@ -36,7 +36,7 @@ void Mesh::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Mesh \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Mesh \"" + filename + "\"", e);
 	}
 }
 

anki/resource/MeshData.cpp

@@ -125,7 +125,7 @@ void MeshData::load(const char* filename)
 	}
 	catch(Exception& e)
 	{
-		throw ANKI_EXCEPTION("File \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("File \"" + filename + "\"", e);
 	}
 }
 

anki/resource/Model.cpp

@@ -57,7 +57,7 @@ void Model::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Model \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Model \"" + filename + "\"", e);
 	}
 }
 

anki/resource/ResourceManager.inl.h

@@ -21,8 +21,8 @@ void ResourceManager<Type>::allocAndLoadRsrc(
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Constructor failed for \"" + filename +
-			"\": " + e.what());
+		throw ANKI_EXCEPTION_R("Constructor failed for \"" + filename +
+			"\"", e);
 	}
 
 	// Load
@@ -32,7 +32,7 @@ void ResourceManager<Type>::allocAndLoadRsrc(
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Cannot load \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Cannot load \"" + filename + "\"", e);
 	}
 }
 
@@ -69,8 +69,8 @@ typename ResourceManager<Type>::Hook& ResourceManager<Type>::load(
 				delete hook;
 			}
 
-			throw ANKI_EXCEPTION("Cannot load \"" +
-				filename + "\": " + e.what());
+			throw ANKI_EXCEPTION_R("Cannot load \"" +
+				filename + "\"", e);
 		}
 
 		hooks.push_back(hook);

anki/resource/ShaderProgramPrePreprocessor.cpp

@@ -245,7 +245,7 @@ void ShaderProgramPrePreprocessor::parseFile(const char* filename)
 	}
 	catch(Exception& e)
 	{
-		throw ANKI_EXCEPTION("Started from \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Started from \"" + filename + "\"", e);
 	}
 }
 

anki/resource/ShaderProgramPrePreprocessor.h

@@ -2,7 +2,6 @@
 #define ANKI_RESOURCE_SHADER_PROGRAM_PRE_PREPROCESSOR_H
 
 #include "anki/util/StdTypes.h"
-#include "anki/util/scanner/Forward.h"
 #include "anki/resource/ShaderProgramCommon.h"
 #include <limits>
 #include <boost/array.hpp>
@@ -12,6 +11,11 @@
 namespace anki {
 
 
+namespace scanner {
+class Scanner;
+}
+
+
 /// Helper class used for shader program loading
 ///
 /// The class fills some of the GLSL spec deficiencies. It adds the include

anki/resource/Skeleton.cpp

@@ -108,7 +108,7 @@ void Skeleton::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Skeleton \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Skeleton \"" + filename + "\"", e);
 	}
 }
 

anki/resource/Skin.cpp

@@ -93,7 +93,7 @@ void Skin::load(const char* filename)
 	  }
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("Skin \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("Skin \"" + filename + "\"", e);
 	}
 }
 

anki/resource/Texture.cpp

@@ -54,7 +54,7 @@ void Texture::load(const char* filename)
 	}
 	catch(std::exception& e)
 	{
-		throw ANKI_EXCEPTION("File \"" + filename + "\": " + e.what());
+		throw ANKI_EXCEPTION_R("File \"" + filename + "\"", e);
 	}
 }
 

anki/util/Exception.cpp

@@ -6,49 +6,22 @@ namespace anki {
 
 
 //==============================================================================
-// Constructor                                                                 =
-//==============================================================================
-Exception::Exception(const std::string& err_, const char* file_, int line_, 
-	const char* func_)
-:	err(err_),
-	file(file_),
-	line(line_),
-	func(func_)
-{}
-
-
-//==============================================================================
-// Copy constructor                                                            =
-//==============================================================================
-Exception::Exception(const Exception& e):
-	err(e.err),
-	file(e.file),
-	line(e.line),
-	func(e.func)
-{}
-
-
-//==============================================================================
-// getInfoStr                                                                  =
-//==============================================================================
-std::string Exception::getInfoStr() const
+std::string Exception::synthErr(const char* error, const char* file,
+	int line, const char* func)
 {
 	std::stringstream ss;
-	
-	ss << '(' << file << ':' << line << ' ' << func << ')';
+	ss << "(" << file << ':' << line << ' ' << func << ") " << error;
 	return ss.str();
 }
 
 
 //==============================================================================
-// what                                                                        =
-//==============================================================================
-const char* Exception::what() const throw()
+Exception::Exception(const char* err, const std::exception& e,
+	const char* file, int line, const char* func)
 {
 	std::stringstream ss;
-	ss << "\n" << getInfoStr() << " AnKi exception: " << err;
-	errWhat = ss.str();
-	return errWhat.c_str();
+	ss << synthErr(error, file, line, func) << ". From here:\n" << e.what();
+	err = ss.str();
 }
 
 

anki/util/Exception.h

@@ -12,33 +12,40 @@ namespace anki {
 /// it using the ANKI_EXCEPTION macro
 class Exception: public std::exception
 {
-	public:
-		/// Constructor
-		Exception(const std::string& err, const char* file = "unknown", 
-			int line = -1, const char* func = "unknown");
-
-		/// Copy constructor
-		Exception(const Exception& e);
-
-		/// Destructor. Do nothing
-		~Exception() throw() {}
-
-		/// @name Accessors
-		/// @{
-		const std::string& getTheError() const {return err;}
-		/// @}
-
-		/// Return the error code formated with the other info
-		virtual const char* what() const throw();
-
-	protected:
-		std::string err;
-		mutable std::string errWhat;
-		const char* file;
-		int line;
-		const char* func;
-
-		std::string getInfoStr() const;
+public:
+	/// Constructor
+	Exception(const char* err, const char* file = "unknown",
+		int line = -1, const char* func = "unknown")
+	{
+		synthErr(err, file, line, func);
+	}
+
+	/// Copy constructor
+	Exception(const Exception& e)
+		: err(e.err)
+	{}
+
+	/// For re-throws
+	Exception(const char* err, const std::exception& e,
+		const char* file = "unknown",
+		int line = -1, const char* func = "unknown");
+
+	/// Destructor. Do nothing
+	~Exception() throw()
+	{}
+
+	/// Implements std::exception::what()
+	const char* what() const throw()
+	{
+		return err.c_str();
+	}
+
+private:
+	std::string err;
+
+	/// XXX
+	std::string synthErr(const char* error, const char* file,
+		int line, const char* func);
 };
 
 
@@ -49,7 +56,10 @@ class Exception: public std::exception
 // Macros                                                                      =
 //==============================================================================
 
-#define ANKI_EXCEPTION(x) Exception(std::string() + x, \
+#define ANKI_EXCEPTION(x) Exception((std::string() + x).c_str(), \
+	__FILE__, __LINE__, __func__)
+
+#define ANKI_EXCEPTION_R(x, e) Exception((std::string() + x).c_str(), e, \
 	__FILE__, __LINE__, __func__)
 
 

anki/util/scanner/Scanner.cpp → anki/util/Scanner.cpp

@@ -1,24 +1,136 @@
-#include "anki/util/scanner/Scanner.h"
-#include "anki/util/scanner/Exception.h"
-#include <fstream>
-#include <iostream>
+#include "anki/util/Scanner.h"
+#include <boost/lexical_cast.hpp>
 #include <cstring>
+#include <iostream>
+#include <iomanip>
 #include <cmath>
-#include <cassert>
 #include <sstream>
-#include <iomanip>
-#include <boost/lexical_cast.hpp>
 
 
-namespace scanner {
+namespace anki { namespace scanner {
+
+
+//==============================================================================
+Exception::Exception(const std::string& err, int errNo_,
+	const std::string& scriptFilename_, int scriptLineNmbr_)
+	: error(err), errNo(errNo_), scriptFilename(scriptFilename_),
+		scriptLineNmbr(scriptLineNmbr_)
+{}
 
-#define SCANNER_EXCEPTION(x) \
-	Exception(std::string() + x, __LINE__, scriptName, lineNmbr)
+
+//==============================================================================
+Exception::Exception(const Exception& e)
+	: std::exception(e), error(e.error), errNo(e.errNo),
+		scriptFilename(e.scriptFilename), scriptLineNmbr(e.scriptLineNmbr)
+{}
 
 
 //==============================================================================
-// statics                                                                     =
+const char* Exception::what() const throw()
+{
+	errWhat = "Scanner exception (#" +
+		boost::lexical_cast<std::string>(errNo) +
+		":" + scriptFilename + ':' +
+		boost::lexical_cast<std::string>(scriptLineNmbr) + "): " + error;
+	return errWhat.c_str();
+}
+
+
+//==============================================================================
+Token::Token(const Token& b)
+	: code(b.code), dataType(b.dataType)
+{
+	switch(b.dataType)
+	{
+		case DT_FLOAT:
+			value.float_ = b.value.float_;
+			break;
+		case DT_INT:
+			value.int_ = b.value.int_;
+			break;
+		case DT_CHAR:
+			value.char_ = b.value.char_;
+			break;
+		case DT_STR:
+			value.string = b.value.string;
+			break;
+	}
+	memcpy(&asString[0], &b.asString[0], sizeof(asString));
+}
+
+
 //==============================================================================
+std::ostream& operator<<(std::ostream& s, const Token& x)
+{
+	const TokenDataVal& val = x.getValue();
+	TokenCode code = x.getCode();
+
+	switch(code)
+	{
+		case TC_COMMENT:
+			s << "comment";
+			break;
+		case TC_NEWLINE:
+			s << "newline";
+			break;
+		case TC_END:
+			s << "end of file";
+			break;
+		case TC_STRING:
+			s << "string \"" << val.getString() << "\"";
+			break;
+		case TC_CHARACTER:
+			s << "char '" << val.getChar() << "' (\"" <<
+				x.getString() << "\")";
+			break;
+		case TC_NUMBER:
+			if(x.getDataType() == DT_FLOAT)
+			{
+				s << "float " << val.getFloat() << " (\"" << x.getString() <<
+					"\")";
+			}
+			else
+			{
+				s << "int " << val.getInt() << " (\"" <<
+					x.getString() << "\")";
+			}
+			break;
+		case TC_IDENTIFIER:
+			s << "identifier \"" << val.getString() << "\"";
+			break;
+		case TC_ERROR:
+			s << "scanner error";
+			break;
+		default:
+			if(code >= TC_KE && code <= TC_KEYWORD)
+			{
+				s << "reserved word \"" << val.getString() << "\"";
+			}
+			else if(code >= TC_SCOPE_RESOLUTION && code <= TC_ASSIGN_OR)
+			{
+				s << "operator no " << (code - TC_SCOPE_RESOLUTION);
+			}
+	}
+
+	return s;
+}
+
+
+//==============================================================================
+std::string Token::getInfoString() const
+{
+	std::stringstream ss;
+	ss << *this;
+	return ss.str();
+}
+
+
+//==============================================================================
+
+#define SCANNER_EXCEPTION(x) \
+	Exception(std::string() + x, __LINE__, scriptName, lineNmbr)
+
+
 char Scanner::eofChar = 0x7F;
 
 
@@ -63,9 +175,6 @@ Scanner::AsciiFlag Scanner::asciiLookupTable [128] = {AC_ERROR};
 
 
 //==============================================================================
-// Constructors                                                                =
-//==============================================================================
-
 Scanner::Scanner(bool newlinesAsWhitespace)
 {
 	strcpy(scriptName, "unnamed-script");
@@ -73,6 +182,7 @@ Scanner::Scanner(bool newlinesAsWhitespace)
 }
 
 
+//==============================================================================
 Scanner::Scanner(const char* filename, bool newlinesAsWhitespace)
 {
 	strcpy(scriptName, "unnamed-script");
@@ -81,6 +191,7 @@ Scanner::Scanner(const char* filename, bool newlinesAsWhitespace)
 }
 
 
+//==============================================================================
 Scanner::Scanner(std::istream& istream_, const char* scriptName_,
 	bool newlinesAsWhitespace)
 {
@@ -90,8 +201,6 @@ Scanner::Scanner(std::istream& istream_, const char* scriptName_,
 }
 
 
-//==============================================================================
-// initAsciiMap                                                                =
 //==============================================================================
 void Scanner::initAsciiMap()
 {
@@ -135,8 +244,6 @@ void Scanner::initAsciiMap()
 }
 
 
-//==============================================================================
-// init                                                                        =
 //==============================================================================
 void Scanner::init(bool newlinesAsWhitespace_)
 {
@@ -154,8 +261,6 @@ void Scanner::init(bool newlinesAsWhitespace_)
 }
 
 
-//==============================================================================
-// getLine                                                                     =
 //==============================================================================
 void Scanner::getLine()
 {
@@ -173,8 +278,6 @@ void Scanner::getLine()
 }
 
 
-//==============================================================================
-// getNextChar                                                                 =
 //==============================================================================
 char Scanner::getNextChar()
 {
@@ -201,8 +304,6 @@ char Scanner::getNextChar()
 }
 
 
-//==============================================================================
-// putBackChar                                                                 =
 //==============================================================================
 char Scanner::putBackChar()
 {
@@ -215,8 +316,6 @@ char Scanner::putBackChar()
 }
 
 
-//==============================================================================
-// getAllPrintAll                                                              =
 //==============================================================================
 void Scanner::getAllPrintAll()
 {
@@ -224,13 +323,11 @@ void Scanner::getAllPrintAll()
 	{
 		getNextToken();
 		std::cout << std::setw(3) << std::setfill('0') << getLineNumber() <<
-			": " << crntToken.getInfoStr() << std::endl;
+			": " << crntToken << std::endl;
 	} while(crntToken.code != TC_END);
 }
 
 
-//==============================================================================
-// loadFile                                                                    =
 //==============================================================================
 void Scanner::loadFile(const char* filename_)
 {
@@ -244,8 +341,6 @@ void Scanner::loadFile(const char* filename_)
 }
 
 
-//==============================================================================
-// loadIstream                                                                 =
 //==============================================================================
 void Scanner::loadIstream(std::istream& istream_, const char* scriptName_)
 {
@@ -268,8 +363,6 @@ void Scanner::loadIstream(std::istream& istream_, const char* scriptName_)
 }
 
 
-//==============================================================================
-// unload                                                                      =
 //==============================================================================
 void Scanner::unload()
 {
@@ -277,8 +370,6 @@ void Scanner::unload()
 }
 
 
-//==============================================================================
-// getNextToken                                                                =
 //==============================================================================
 const Token& Scanner::getNextToken()
 {
@@ -385,8 +476,7 @@ const Token& Scanner::getNextToken()
 	return crntToken;
 }
 
-//==============================================================================
-// checkWord                                                                   =
+
 //==============================================================================
 void Scanner::checkWord()
 {
@@ -429,8 +519,6 @@ void Scanner::checkWord()
 }
 
 
-//==============================================================================
-// checkComment                                                                =
 //==============================================================================
 void Scanner::checkComment()
 {
@@ -499,8 +587,6 @@ void Scanner::checkComment()
 }
 
 
-//==============================================================================
-// checkNumber                                                                 =
 //==============================================================================
 void Scanner::checkNumber()
 {
@@ -812,8 +898,6 @@ void Scanner::checkNumber()
 }
 
 
-//==============================================================================
-// checkString                                                                 =
 //==============================================================================
 void Scanner::checkString()
 {
@@ -904,8 +988,6 @@ void Scanner::checkString()
 }
 
 
-//==============================================================================
-// checkChar                                                                   =
 //==============================================================================
 void Scanner::checkChar()
 {
@@ -933,10 +1015,9 @@ void Scanner::checkChar()
 	{
 		ch = getNextChar();
 		*tmpStr++ = ch;
-		if(ch=='\0' || ch==eofChar) //check again after the \.
+		if(ch == '\0' || ch == eofChar) //check again after the \.
 		{
 			throw SCANNER_EXCEPTION("Newline in constant");
-			return;
 		}
 
 		switch (ch)
@@ -971,6 +1052,9 @@ void Scanner::checkChar()
 			case '\?':
 				ch0 = '\?';
 				break;
+			case 'r':
+				ch0 = '\r';
+				break;
 			default:
 				ch0 = ch;
 				throw SCANNER_EXCEPTION("Unrecognized escape character \'\\" +
@@ -996,8 +1080,6 @@ void Scanner::checkChar()
 }
 
 
-//==============================================================================
-// checkSpecial                                                                =
 //==============================================================================
 void Scanner::checkSpecial()
 {
@@ -1270,4 +1352,4 @@ void Scanner::checkSpecial()
 }
 
 
-} // end namesapce
+}} // end namespaces

anki/util/Scanner.h

@@ -0,0 +1,365 @@
+#ifndef ANKI_UTIL_SCANNER_H
+#define ANKI_UTIL_SCANNER_H
+
+#include <exception>
+#include <boost/array.hpp>
+#include <iosfwd>
+#include <fstream>
+
+
+namespace anki { namespace scanner {
+
+
+/// Scanner exception
+class Exception: public std::exception
+{
+public:
+	/// Constructor
+	Exception(const std::string& err, int errNo,
+		const std::string& scriptFilename, int scriptLineNmbr);
+
+	/// Copy constructor
+	Exception(const Exception& e);
+
+	/// Destructor. Do nothing
+	~Exception() throw()
+	{}
+
+	/// Return the error code
+	virtual const char* what() const throw();
+
+private:
+	std::string error;
+	int errNo; ///< Error number
+	std::string scriptFilename;
+	int scriptLineNmbr;
+	mutable std::string errWhat;
+};
+
+
+/// The max allowed length of a script line
+const int MAX_SCRIPT_LINE_LEN = 1024;
+
+
+/// The TokenCode is an enum that defines the Token type
+enum TokenCode
+{
+	// general codes
+	TC_ERROR, TC_END, TC_COMMENT, TC_NUMBER, TC_CHARACTER, TC_STRING,
+	TC_IDENTIFIER, TC_NEWLINE,
+
+	// keywords listed by strlen (dummy keywords at the moment)
+	TC_KE,
+	TC_KEY,
+	TC_KEYW,
+	TC_KEYWO,
+	TC_KEYWOR,
+	TC_KEYWORD,
+
+	// operators
+	TC_SCOPE_RESOLUTION, TC_L_SQ_BRACKET, TC_R_SQ_BRACKET, TC_L_PAREN,
+		TC_R_PAREN,
+	TC_DOT, TC_POINTER_TO_MEMBER, TC_L_BRACKET, TC_R_BRACKET, TC_COMMA,
+	TC_PERIOD, TC_UPDOWNDOT, TC_QUESTIONMARK, TC_SHARP, TC_EQUAL,
+	TC_NOT_EQUAL, TC_LESS, TC_GREATER, TC_LESS_EQUAL, TC_GREATER_EQUAL,
+	TC_LOGICAL_OR, TC_LOGICAL_AND, TC_PLUS, TC_MINUS, TC_STAR,
+	TC_BSLASH, TC_NOT, TC_BITWISE_AND, TC_BITWISE_OR, TC_UNARAY_COMPLEMENT,
+	TC_MOD, TC_XOR, TC_INC, TC_DEC, TC_SHL,
+	TC_SHR, TC_ASSIGN, TC_ASSIGN_ADD, TC_ASSIGN_SUB, TC_ASSIGN_MUL,
+	TC_ASSIGN_DIV, TC_ASSIGN_MOD, TC_ASSIGN_SHL, TC_ASSIGN_SHR, TC_ASSIGN_AND,
+	TC_ASSIGN_XOR, TC_ASSIGN_OR, TC_BACK_SLASH
+}; // end enum TokenCode
+
+
+/// The value of Token::dataType
+enum DataType
+{
+	DT_FLOAT,
+	DT_INT,
+	DT_CHAR,
+	DT_STR
+};
+
+
+/// Used inside the Token, its a variant that holds the data of the Token
+class TokenDataVal
+{
+	friend class Scanner;
+	friend class Token;
+
+public:
+	/// @name Accessors
+	/// @{
+
+	/// Access the data as C char
+	char getChar() const
+	{
+		return char_;
+	}
+
+	/// Access the data as unsigned int
+	unsigned long getInt() const
+	{
+		return int_;
+	}
+
+	/// Access the data as double
+	double getFloat() const
+	{
+		return float_;
+	}
+
+	/// Access the data as C string
+	const char* getString() const
+	{
+		return string;
+	}
+	/// @}
+
+private:
+	/// The data as unnamed union
+	union
+	{
+		char char_;
+		unsigned long int_;
+		double float_;
+		/// Points to @ref Token::asString if the token is string or
+		/// identifier
+		char* string;
+	};
+};
+
+
+/// The Token class
+class Token
+{
+	friend class Scanner;
+
+public:
+	Token()
+		: code(TC_ERROR)
+	{}
+
+	Token(const Token& b);
+
+	/// @name accessors
+	/// @{
+	const char* getString() const
+	{
+		return &asString[0];
+	}
+
+	TokenCode getCode() const
+	{
+		return code;
+	}
+
+	DataType getDataType() const
+	{
+		return dataType;
+	}
+
+	const TokenDataVal& getValue() const
+	{
+		return value;
+	}
+	/// @}
+
+	std::string getInfoString() const;
+
+	friend std::ostream& operator<<(std::ostream& s,
+		const Token& x);
+
+private:
+	boost::array<char, MAX_SCRIPT_LINE_LEN> asString;
+	TokenCode code; ///< The first thing you should know about a token
+
+	/// Additional info in case @ref code is @ref TC_NUMBER
+	DataType dataType;
+	TokenDataVal value; ///< A value variant
+};
+
+
+/// C++ Tokenizer
+///
+/// The Scanner loads a file or an already loaded iostream and extracts the
+/// tokens. The script must be in C++ format. The class does not make any kind
+/// of memory allocations so it can be fast.
+class Scanner
+{
+public:
+	/// Constructor #1
+	/// @param newlinesAsWhitespace @see newlinesAsWhitespace
+	Scanner(bool newlinesAsWhitespace = true);
+
+	/// Constructor #2
+	/// @see loadFile
+	/// @param newlinesAsWhitespace @see newlinesAsWhitespace
+	/// @exception Exception
+	Scanner(const char* filename, bool newlinesAsWhitespace = true);
+
+	/// Constructor #3
+	/// @see loadIstream
+	/// @param newlinesAsWhitespace @see newlinesAsWhitespace
+	/// @exception Exception
+	Scanner(std::istream& istream_,
+		const char* scriptName_ = "unamed-istream",
+		bool newlinesAsWhitespace = true);
+
+	/// It only unloads the file if file is chosen
+	~Scanner()
+	{
+		unload();
+	}
+
+	/// Load a file to extract tokens
+	/// @param filename The filename of the file to read
+	/// @exception Exception
+	void loadFile(const char* filename);
+
+	/// Load a STL istream to extract tokens
+	/// @param istream_ The stream from where to read
+	/// @param scriptName_ The name of the stream. For error reporting
+	/// @exception Exception
+	void loadIstream(std::istream& istream_,
+		const char* scriptName_ = "unamed-istream");
+
+	/// Extracts all tokens and prints them. Used for debugging
+	void getAllPrintAll();
+
+	/// Get the next token from the stream. Its virtual and you can
+	/// override it
+	/// @return The next Token
+	/// @exception Exception
+	virtual const Token& getNextToken();
+
+	/// Accessor for the current token
+	/// @return The current Token
+	const Token& getCrntToken() const
+	{
+		return crntToken;
+	}
+
+	/// Get the name of the input stream
+	const char* getScriptName() const
+	{
+		return scriptName;
+	}
+
+	/// Get the current line the Scanner is processing
+	int getLineNumber() const
+	{
+		return lineNmbr;
+	}
+
+protected:
+	/// Every char in the Ascii table is binded with one characteristic
+	/// code type. This helps the scanning
+	enum AsciiFlag
+	{
+		AC_ERROR = 0,
+		AC_EOF = 1,
+		AC_LETTER = 2,
+		AC_DIGIT = 4,
+		AC_SPECIAL = 8,
+		AC_WHITESPACE = 16,
+		AC_QUOTE = 32,
+		AC_DOUBLEQUOTE = 64,
+		AC_ACCEPTABLE_IN_COMMENTS = 128 ///< Only accepted in comments
+	};
+
+	/// Reserved words like "int" "const" etc. Currently the reserved words
+	/// list is being populated with dummy data
+	struct ResWord
+	{
+		const char* string;
+		TokenCode   code;
+	};
+
+	static char eofChar; ///< Special end of file character
+
+	/// The array contains one AsciiFlag for every symbol of the ASCII table
+	static AsciiFlag asciiLookupTable[];
+
+	/// @name Reserved words
+	/// Groups of ResWord grouped by the length of the ResWord::string
+	/// @{
+	static ResWord rw2[], rw3[], rw4[], rw5[], rw6[], rw7[];
+	/// @}
+
+	/// The array contains all the groups of ResWord
+	static ResWord* rwTable[];
+
+	Token crntToken; ///< The current token
+	/// In contains the current line's text
+	char line[MAX_SCRIPT_LINE_LEN];
+	char* pchar; ///< Points somewhere to @ref line
+	int lineNmbr; ///< The number of the current line
+
+	/// Treat newlines as whitespace. If false means that the Scanner
+	/// returns (among others) newline tokens
+	bool newlinesAsWhitespace;
+
+	/// Commented lines number
+	/// Used to keep track of the newlines in multiline comments so we can
+	/// then return the correct number of newlines
+	/// in case of newlinesAsWhitespace is false
+	int commentedLines;
+
+	/// @name Input
+	/// @{
+
+	/// The file stream. Used if the @ref Scanner is initiated using
+	/// @ref loadFile
+	std::ifstream inFstream;
+	/// Points to either @ref inFstream or an external std::istream
+	std::istream* inStream;
+	/// The name of the input stream. Mainly used for error messaging
+	char scriptName[512];
+	/// @}
+
+	/// @name Checkers
+	/// @{
+	void checkWord();
+	void checkComment();
+	void checkNumber();
+	void checkString();
+	void checkChar();
+	void checkSpecial();
+	/// @}
+
+	/// It reads a new line from the iostream and it points @ref pchar to
+	/// the beginning of that line
+	void getLine();
+
+	/// Get the next char from the @ref line. If @ref line empty then get
+	/// new line. It returns '\\0' if we are in the
+	/// end of the line
+	char getNextChar();
+
+	/// Put the char that @ref getNextChar got back to the current line
+	char putBackChar();
+
+	/// Initializes the asciiLookupTable. It runs only once in the
+	/// construction of the first Scanner @see Scanner()
+	static void initAsciiMap();
+
+	/// A function to save us from typing
+	static AsciiFlag& lookupAscii(char c)
+	{
+		return asciiLookupTable[static_cast<int>(c)];
+	}
+
+	/// Common initialization code
+	void init(bool newlinesAsWhitespace_);
+
+	/// Unloads the file
+	void unload();
+};
+
+
+
+}} // end namespaces
+
+
+#endif

anki/util/scanner/Common.h

@@ -1,13 +0,0 @@
-#ifndef SCANNER_COMMON_H
-#define SCANNER_COMMON_H
-
-
-namespace scanner {
-
-/// The max allowed length of a script line
-const int MAX_SCRIPT_LINE_LEN = 1024;
-
-}
-
-
-#endif

anki/util/scanner/Exception.cpp

@@ -1,44 +0,0 @@
-#include "anki/util/scanner/Exception.h"
-#include <boost/lexical_cast.hpp>
-
-
-namespace scanner {
-
-
-//==============================================================================
-// Constructor                                                                 =
-//==============================================================================
-Exception::Exception(const std::string& err, int errNo_,
-	const std::string& scriptFilename_, int scriptLineNmbr_)
-:	error(err),
-	errNo(errNo_),
-	scriptFilename(scriptFilename_),
-	scriptLineNmbr(scriptLineNmbr_)
-{}
-
-
-//==============================================================================
-// Copy constructor                                                            =
-//==============================================================================
-Exception::Exception(const Exception& e)
-:	error(e.error),
-	errNo(e.errNo),
-	scriptFilename(e.scriptFilename),
-	scriptLineNmbr(e.scriptLineNmbr)
-{}
-
-
-//==============================================================================
-// what                                                                        =
-//==============================================================================
-const char* Exception::what() const throw()
-{
-	errWhat = "Scanner exception (#" +
-		boost::lexical_cast<std::string>(errNo) +
-		":" + scriptFilename + ':' +
-		boost::lexical_cast<std::string>(scriptLineNmbr) + "): " + error;
-	return errWhat.c_str();
-}
-
-
-} // end namespace

anki/util/scanner/Exception.h

@@ -1,39 +0,0 @@
-#ifndef SCANNER_EXCEPTION_H
-#define SCANNER_EXCEPTION_H
-
-#include <exception>
-#include <string>
-
-
-namespace scanner {
-
-
-class Exception: public std::exception
-{
-	public:
-		/// Constructor
-		Exception(const std::string& err, int errNo,
-			const std::string& scriptFilename, int scriptLineNmbr);
-
-		/// Copy constructor
-		Exception(const Exception& e);
-
-		/// Destructor. Do nothing
-		~Exception() throw() {}
-
-		/// Return the error code
-		virtual const char* what() const throw();
-
-	private:
-		std::string error;
-		int errNo; ///< Error number
-		std::string scriptFilename;
-		int scriptLineNmbr;
-		mutable std::string errWhat;
-};
-
-
-} // End namespace
-
-
-#endif

anki/util/scanner/Forward.h

@@ -1,15 +0,0 @@
-#ifndef SCANNER_FORWARD_H
-#define SCANNER_FORWARD_H
-
-
-namespace scanner {
-
-
-class Scanner;
-class Token;
-
-
-} // end namespace
-
-
-#endif

anki/util/scanner/Scanner.h

@@ -1,189 +0,0 @@
-#ifndef SCANNER_SCANNER_H
-#define SCANNER_SCANNER_H
-
-#include "anki/util/scanner/Token.h"
-#include <fstream>
-
-
-namespace scanner {
-
-
-/// C++ Tokenizer
-///
-/// The Scanner loads a file or an already loaded iostream and extracts the
-/// tokens. The script must be in C++ format. The class does not make any kind
-/// of memory allocations so it can be fast.
-class Scanner
-{
-	public:
-		/// Constructor #1
-		/// @param newlinesAsWhitespace @see newlinesAsWhitespace
-		Scanner(bool newlinesAsWhitespace = true);
-
-		/// Constructor #2
-		/// @see loadFile
-		/// @param newlinesAsWhitespace @see newlinesAsWhitespace
-		/// @exception Exception
-		Scanner(const char* filename, bool newlinesAsWhitespace = true);
-
-		/// Constructor #3
-		/// @see loadIstream
-		/// @param newlinesAsWhitespace @see newlinesAsWhitespace
-		/// @exception Exception
-		Scanner(std::istream& istream_,
-			const char* scriptName_ = "unamed-istream",
-			bool newlinesAsWhitespace = true);
-
-		/// It only unloads the file if file is chosen
-		~Scanner() {unload();}
-
-		/// Load a file to extract tokens
-		/// @param filename The filename of the file to read
-		/// @exception Exception
-		void loadFile(const char* filename);
-
-		/// Load a STL istream to extract tokens
-		/// @param istream_ The stream from where to read
-		/// @param scriptName_ The name of the stream. For error reporting
-		/// @exception Exception
-		void loadIstream(std::istream& istream_,
-			const char* scriptName_ = "unamed-istream");
-
-		/// Extracts all tokens and prints them. Used for debugging
-		void getAllPrintAll();
-
-		/// Get the next token from the stream. Its virtual and you can
-		/// override it
-		/// @return The next Token
-		/// @exception Exception
-		virtual const Token& getNextToken();
-
-		/// Accessor for the current token
-		/// @return The current Token
-		const Token& getCrntToken() const
-		{
-			return crntToken;
-		}
-
-		/// Get the name of the input stream
-		const char* getScriptName() const
-		{
-			return scriptName;
-		}
-
-		/// Get the current line the Scanner is processing
-		int getLineNumber() const
-		{
-			return lineNmbr;
-		}
-
-	protected:
-		/// Every char in the Ascii table is binded with one characteristic
-		/// code type. This helps the scanning
-		enum AsciiFlag
-		{
-			AC_ERROR = 0,
-			AC_EOF = 1,
-			AC_LETTER = 2,
-			AC_DIGIT = 4,
-			AC_SPECIAL = 8,
-			AC_WHITESPACE = 16,
-			AC_QUOTE = 32,
-			AC_DOUBLEQUOTE = 64,
-			AC_ACCEPTABLE_IN_COMMENTS = 128 ///< Only accepted in comments
-		};
-
-		/// Reserved words like "int" "const" etc. Currently the reserved words
-		/// list is being populated with dummy data
-		struct ResWord
-		{
-			const char* string;
-			TokenCode   code;
-		};
-
-		static char eofChar; ///< Special end of file character
-
-		/// The array contains one AsciiFlag for every symbol of the ASCII table
-		static AsciiFlag asciiLookupTable[];
-
-		/// @name Reserved words
-		/// Groups of ResWord grouped by the length of the ResWord::string
-		/// @{
-		static ResWord rw2[], rw3[], rw4[], rw5[], rw6[], rw7[];
-		/// @}
-
-		/// The array contains all the groups of ResWord
-		static ResWord* rwTable[];
-
-		Token crntToken; ///< The current token
-		/// In contains the current line's text
-		char  line[MAX_SCRIPT_LINE_LEN];
-		char* pchar; ///< Points somewhere to @ref line
-		int   lineNmbr; ///< The number of the current line
-
-		/// Treat newlines as whitespace. If false means that the Scanner
-		/// returns (among others) newline tokens
-		bool newlinesAsWhitespace;
-
-		/// Commented lines number
-		/// Used to keep track of the newlines in multiline comments so we can
-		/// then return the correct number of newlines
-		/// in case of newlinesAsWhitespace is false
-		int commentedLines;
-
-		/// @name Input
-		/// @{
-
-		/// The file stream. Used if the @ref Scanner is initiated using
-		/// @ref loadFile
-		std::ifstream inFstream;
-		/// Points to either @ref inFstream or an external std::istream
-		std::istream* inStream;
-		/// The name of the input stream. Mainly used for error messaging
-		char scriptName[512];
-		/// @}
-
-		/// @name Checkers
-		/// @{
-		void checkWord();
-		void checkComment();
-		void checkNumber();
-		void checkString();
-		void checkChar();
-		void checkSpecial();
-		/// @}
-
-		/// It reads a new line from the iostream and it points @ref pchar to
-		/// the beginning of that line
-		void getLine();
-
-		/// Get the next char from the @ref line. If @ref line empty then get
-		/// new line. It returns '\\0' if we are in the
-		/// end of the line
-		char getNextChar();
-
-		/// Put the char that @ref getNextChar got back to the current line
-		char putBackChar();
-
-		/// Initializes the asciiLookupTable. It runs only once in the
-		/// construction of the first Scanner @see Scanner()
-		static void initAsciiMap();
-
-		/// A function to save us from typing
-		static AsciiFlag& lookupAscii(char c)
-		{
-			return asciiLookupTable[static_cast<int>(c)];
-		}
-
-		/// Common initialization code
-		void init(bool newlinesAsWhitespace_);
-
-		/// Unloads the file
-		void unload();
-};
-
-
-} // end namespace
-
-
-#endif

anki/util/scanner/Token.cpp

@@ -1,99 +0,0 @@
-#include "anki/util/scanner/Token.h"
-#include <cstring>
-#include <cstdio>
-#include <iostream>
-#include <boost/lexical_cast.hpp>
-
-
-namespace scanner {
-
-
-//==============================================================================
-// Copy constructor                                                            =
-//==============================================================================
-Token::Token(const Token& b)
-:	code(b.code),
-	dataType(b.dataType)
-{
-	switch(b.dataType)
-	{
-		case DT_FLOAT:
-			value.float_ = b.value.float_;
-			break;
-		case DT_INT:
-			value.int_ = b.value.int_;
-			break;
-		case DT_CHAR:
-			value.char_ = b.value.char_;
-			break;
-		case DT_STR:
-			value.string = b.value.string;
-			break;
-	}
-	memcpy(&asString[0], &b.asString[0], sizeof(asString));
-}
-
-
-//==============================================================================
-// getInfoStr                                                                  =
-//==============================================================================
-std::string Token::getInfoStr() const
-{
-	char tokenInfoStr[512];
-	switch(code)
-	{
-		case TC_COMMENT:
-			return "comment";
-		case TC_NEWLINE:
-			return "newline";
-		case TC_END:
-			return "end of file";
-		case TC_STRING:
-			sprintf(tokenInfoStr, "string \"%s\"", value.string);
-			break;
-		case TC_CHARACTER:
-			sprintf(tokenInfoStr, "char '%c' (\"%s\")", value.char_,
-				&asString[0]);
-			break;
-		case TC_NUMBER:
-			if(dataType == DT_FLOAT)
-			{
-				sprintf(tokenInfoStr, "float %f or %e (\"%s\")", value.float_,
-					value.float_, &asString[0]);
-			}
-			else
-				sprintf(tokenInfoStr, "int %lu (\"%s\")", value.int_,
-					&asString[0]);
-			break;
-		case TC_IDENTIFIER:
-			sprintf(tokenInfoStr, "identifier \"%s\"", value.string);
-			break;
-		case TC_ERROR:
-			return "scanner error";
-			break;
-		default:
-			if(code >= TC_KE && code <= TC_KEYWORD)
-			{
-				sprintf(tokenInfoStr, "reserved word \"%s\"", value.string);
-			}
-			else if(code>=TC_SCOPE_RESOLUTION && code<=TC_ASSIGN_OR)
-			{
-				sprintf(tokenInfoStr, "operator no %d",
-					code - TC_SCOPE_RESOLUTION);
-			}
-	}
-
-	return tokenInfoStr;
-}
-
-
-//==============================================================================
-// print                                                                       =
-//==============================================================================
-void Token::print() const
-{
-	std::cout << "Token: " << getInfoStr() << std::endl;
-}
-
-
-} // end namespace

anki/util/scanner/Token.h

@@ -1,115 +0,0 @@
-#ifndef SCANNER_TOKEN_H
-#define SCANNER_TOKEN_H
-
-#include "anki/util/scanner/Common.h"
-#include <string>
-#include <boost/array.hpp>
-
-
-namespace scanner {
-
-
-/// The TokenCode is an enum that defines the Token type
-enum TokenCode
-{
-	// general codes
-	TC_ERROR, TC_END, TC_COMMENT, TC_NUMBER, TC_CHARACTER, TC_STRING,
-	TC_IDENTIFIER, TC_NEWLINE,
-
-	// keywords listed by strlen (dummy keywords at the moment)
-	TC_KE,
-	TC_KEY,
-	TC_KEYW,
-	TC_KEYWO,
-	TC_KEYWOR,
-	TC_KEYWORD,
-
-	// operators
-	TC_SCOPE_RESOLUTION, TC_L_SQ_BRACKET, TC_R_SQ_BRACKET, TC_L_PAREN, 
-		TC_R_PAREN,
-	TC_DOT, TC_POINTER_TO_MEMBER, TC_L_BRACKET, TC_R_BRACKET, TC_COMMA,
-	TC_PERIOD, TC_UPDOWNDOT, TC_QUESTIONMARK, TC_SHARP, TC_EQUAL,
-	TC_NOT_EQUAL, TC_LESS, TC_GREATER, TC_LESS_EQUAL, TC_GREATER_EQUAL,
-	TC_LOGICAL_OR, TC_LOGICAL_AND, TC_PLUS, TC_MINUS, TC_STAR,
-	TC_BSLASH, TC_NOT, TC_BITWISE_AND, TC_BITWISE_OR, TC_UNARAY_COMPLEMENT,
-	TC_MOD, TC_XOR, TC_INC, TC_DEC, TC_SHL,
-	TC_SHR, TC_ASSIGN, TC_ASSIGN_ADD, TC_ASSIGN_SUB, TC_ASSIGN_MUL,
-	TC_ASSIGN_DIV, TC_ASSIGN_MOD, TC_ASSIGN_SHL, TC_ASSIGN_SHR, TC_ASSIGN_AND,
-	TC_ASSIGN_XOR, TC_ASSIGN_OR, TC_BACK_SLASH
-}; // end enum TokenCode
-
-
-/// The value of Token::dataType
-enum DataType
-{
-	DT_FLOAT,
-	DT_INT,
-	DT_CHAR,
-	DT_STR
-};
-
-
-/// Used inside the Token, its a variant that holds the data of the Token
-class TokenDataVal
-{
-	friend class Scanner;
-	friend class Token;
-
-	public:
-		/// @name Accessors
-		/// @{
-		char getChar() const {return char_;} ///< Access the data as C char
-		/// Access the data as unsigned int
-		unsigned long getInt() const {return int_;}
-		double getFloat() const {return float_;} ///< Access the data as double
-		/// Access the data as C string
-		const char* getString() const {return string;}
-		/// @}
-
-	private:
-		/// The data as unnamed union
-		union
-		{
-			char char_;
-			unsigned long int_;
-			double float_;
-			/// Points to @ref Token::asString if the token is string or
-			/// identifier
-			char* string;
-		};
-}; // end class TokenDataVal
-
-
-/// The Token class
-class Token
-{
-	friend class Scanner;
-
-	public:
-		Token(): code(TC_ERROR) {}
-		Token(const Token& b);
-		/// Get a string with the info of the token
-		std::string getInfoStr() const;
-		void print() const; ///< Print info of the token
-
-		/// @name accessors
-		/// @{
-		const char* getString() const {return &asString[0];}
-		TokenCode getCode() const {return code;}
-		DataType getDataType() const {return dataType;}
-		const TokenDataVal& getValue() const {return value;}
-		/// @}
-
-	private:
-		boost::array<char, MAX_SCRIPT_LINE_LEN> asString;
-		TokenCode code; ///< The first thing you should know about a token
-		/// Additional info in case @ref code is @ref TC_NUMBER
-		DataType dataType;
-		TokenDataVal value; ///< A value variant
-};
-
-
-} // end namespace
-
-
-#endif