beefytech.Beef/IDEHelper/Compiler/BfParser.cpp

#pragma warning(disable:4996)


#include "BfParser.h"
#include "BfReducer.h"
#include "BfPrinter.h"
#include "BfDefBuilder.h"
#include "BfCompiler.h"
#include "BfSourceClassifier.h"
#include "BfSourcePositionFinder.h"
#include <sstream>
#include "BeefySysLib/util/PerfTimer.h"
#include "BeefySysLib/util/BeefPerf.h"
#include "BeefySysLib/util/UTF8.h"
#include "BfAutoComplete.h"
#include "BfResolvePass.h"
#include "BfElementVisitor.h"
#include "BeefySysLib/util/UTF8.h"

extern "C"
{
#include "BeefySysLib/third_party/utf8proc/utf8proc.h"
}

#include "BeefySysLib/util/AllocDebug.h"

USING_NS_BF;

static bool IsWhitespace(char c)
{
	return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\r');
}

static bool IsWhitespaceOrPunctuation(char c)
{
	switch (c)
	{
	case ',':
	case ';':
	case ':':
	case '(':
	case ')':
	case '[':
	case ']':
	case '{':
	case '}':
	case '<':
	case '>':
	case '/':
	case '-':
	case '=':
	case '+':
	case '!':
	case '%':
	case '&':
	case '|':
	case '#':
	case '@':
	case '`':
	case '^':
	case '~':
	case '*':
	case '?':
	case '\n':
	case ' ':
	case '\t':
	case '\v':
	case '\f':
	case '\r':
		return true;
	default:
		return false;
	}
}

BfParser* BfParserData::ToParser()
{
	if (mUniqueParser != NULL)
	{
		BF_ASSERT(mUniqueParser->mOrigSrcLength >= 0);
		BF_ASSERT((mUniqueParser->mCursorIdx >= -1) || (mUniqueParser->mCursorIdx <= mUniqueParser->mOrigSrcLength));
	}
	return mUniqueParser;
}

//////////////////////////////////////////////////////////////////////////

/// raw_null_ostream - A raw_ostream that discards all output.
/*class debug_ostream : public llvm::raw_ostream
{
/// write_impl - See raw_ostream::write_impl.
void write_impl(const char *Ptr, size_t size) override
{
char str[256] = {0};
memcpy(str, Ptr, std::min((int)size, 255));
OutputDebugStr(str);
}

/// current_pos - Return the current position within the stream, not
/// counting the bytes currently in the buffer.
uint64_t current_pos() const override
{
return 0;
}
};*/

//////////////////////////////////////////////////////////////////////////

BfParserCache* Beefy::gBfParserCache = NULL;

bool BfParserCache::DataEntry::operator==(const LookupEntry& lookup) const
{
	if ((mParserData->mFileName == lookup.mFileName) &&
		(mParserData->mSrcLength == lookup.mSrcLength) &&
		(memcmp(mParserData->mSrc, lookup.mSrc, lookup.mSrcLength) == 0))
	{
		for (auto& setDefine : mParserData->mDefines_Def)
			if (!lookup.mProject->mPreprocessorMacros.Contains(setDefine))
				return false;
		for (auto& setDefine : mParserData->mDefines_NoDef)
			if (lookup.mProject->mPreprocessorMacros.Contains(setDefine))
				return false;

		return true;
	}
	return false;
}

BfParserCache::BfParserCache()
{
	mRefCount = 0;
}

BfParserCache::~BfParserCache()
{
	for (auto& entry : mEntries)
	{
		BF_ASSERT(entry.mParserData->mRefCount == 0);
		delete entry.mParserData;
	}
}

void BfParserCache::ReportMemory(MemReporter* memReporter)
{
	int srcLen = 0;
	int allocBytesUsed = 0;
	int largeAllocs = 0;

	for (auto& entry : mEntries)
	{
		auto parserData = entry.mParserData;
		parserData->ReportMemory(memReporter);
		srcLen += parserData->mSrcLength;
		allocBytesUsed += (int)(parserData->mAlloc.mPages.size() * BfAstAllocManager::PAGE_SIZE);
		largeAllocs += parserData->mAlloc.mLargeAllocSizes;
	}

	int allocPages = 0;
	int usedPages = 0;
	mAstAllocManager.GetStats(allocPages, usedPages);
	
	OutputDebugStrF("Parsers: %d  Chars: %d  UsedAlloc: %dk  BytesPerChar: %d  SysAllocPages: %d  SysUsedPages: %d (%dk)  LargeAllocs: %dk\n", (int)mEntries.size(), srcLen, allocBytesUsed / 1024,
		allocBytesUsed / BF_MAX(1, srcLen), allocPages, usedPages, (usedPages * BfAstAllocManager::PAGE_SIZE) / 1024, largeAllocs / 1024);

	//memReporter->AddBumpAlloc("BumpAlloc", mAstAllocManager);
}

void BfParserData::ReportMemory(MemReporter* memReporter)
{
	memReporter->Add("JumpTable", mJumpTableSize * sizeof(BfLineStartEntry));
	memReporter->Add("Source", mSrcLength);
	memReporter->AddBumpAlloc("AstAlloc", mAlloc);
}

static int DecodeInt(uint8* buf, int& idx)
{
	int value = 0;
	int shift = 0;
	int curByte;
	do
	{
		curByte = buf[idx++];
		value |= ((curByte & 0x7f) << shift);
		shift += 7;

	} while (curByte >= 128);
	// Sign extend negative numbers.
	if (((curByte & 0x40) != 0) && (shift < 64))
		value |= ~0LL << shift;
	return value;
}

static int gCurDataId = 0;
BfParserData::BfParserData()
{		
	mDataId = (int)BfpSystem_InterlockedExchangeAdd32((uint32*)&gCurDataId, 1) + 1;

	mHash = 0;
	mRefCount = -1;
	mJumpTable = NULL;
	mJumpTableSize = 0;
	mFailed = false;	
	mCharIdData = NULL;
	mUniqueParser = NULL;
	mDidReduce = false;
}

BfParserData::~BfParserData()
{
	delete[] mJumpTable;
	delete[] mCharIdData;
}

int BfParserData::GetCharIdAtIndex(int findIndex)
{
	if (mCharIdData == NULL)
		return findIndex;

	int encodeIdx = 0;
	int charId = 1;
	int charIdx = 0;
	while (true)
	{
		int cmd = DecodeInt(mCharIdData, encodeIdx);
		if (cmd > 0)
			charId = cmd;
		else
		{
			int spanSize = -cmd;
			if ((findIndex >= charIdx) && (findIndex < charIdx + spanSize))
				return charId + (findIndex - charIdx);
			charId += spanSize;
			charIdx += spanSize;

			if (cmd == 0)
				return -1;
		}
	}
}

void BfParserData::GetLineCharAtIdx(int idx, int& line, int& lineChar)
{
	auto* jumpEntry = mJumpTable + (idx / PARSER_JUMPTABLE_DIVIDE);
	if (jumpEntry->mCharIdx > idx)
		jumpEntry--;

	line = jumpEntry->mLineNum;
	lineChar = 0;
	int curSrcPos = jumpEntry->mCharIdx;

	while (curSrcPos < idx)
	{
		if (mSrc[curSrcPos] == '\n')
		{
			line++;
			lineChar = 0;
		}
		else
		{
			lineChar++;
		}

		curSrcPos++;
	}
}

bool BfParserData::IsUnwarnedAt(BfAstNode* node)
{
	if (mUnwarns.empty())
		return false;
	auto unwarnItr = mUnwarns.upper_bound(node->GetSrcStart());
	if (unwarnItr == mUnwarns.begin())
		return false;
	unwarnItr--;

	int unwarnIdx = *unwarnItr;

	int checkIdx = node->GetSrcStart();
	int lineCount = 0;
	while (checkIdx > 0)
	{
		checkIdx--;
		if (checkIdx < unwarnIdx)
			return true;
		if (mSrc[checkIdx] == '\n')
		{
			lineCount++;
			// #unwarn must be immediately preceding the start of the statement containing this node
			if (lineCount == 2)
				return false;
		}
	}

	return true;
}

bool BfParserData::IsWarningEnabledAtSrcIndex(int warningNumber, int srcIdx)
{
	bool enabled = true; //CDH TODO if/when we add warning level support, this default will change based on the warning number and the general project warning level setting
	int lastUnwarnPos = 0;

	for (const auto& it : mWarningEnabledChanges)
	{
		if (it.mKey > srcIdx)
			break;
		if (it.mValue.mWarningNumber == warningNumber)
			enabled = it.mValue.mEnable;
		if (it.mValue.mWarningNumber == -1)
			lastUnwarnPos = -1;
	}
	return enabled;
}

void BfParserData::Deref()
{
	mRefCount--;
	BF_ASSERT(mRefCount >= 0);
	if (mRefCount == 0)
	{
		AutoCrit autoCrit(gBfParserCache->mCritSect);
		BfParserCache::DataEntry dataEntry;
		dataEntry.mParserData = this;
		bool didRemove = gBfParserCache->mEntries.Remove(dataEntry);
		BF_ASSERT(didRemove);
		delete this;
	}
}

//////////////////////////////////////////////////////////////////////////

static int gParserCount = 0;

BfParser::BfParser(BfSystem* bfSystem, BfProject* bfProject) : BfSource(bfSystem)
{
	BfLogSys(bfSystem, "BfParser::BfParser %08X\n", this);

	gParserCount++;	

	mUsingCache = false;
	mParserData = NULL;
	mAwaitingDelete = false;
	mScanOnly = false;
	mCompleteParse = false;
	mJumpTable = NULL;
	mProject = bfProject;
	mPassInstance = NULL;
	mPassInstance = NULL;
	mPrevRevision = NULL;
	mNextRevision = NULL;
	mOrigSrcLength = 0;
	mSrcAllocSize = -1;
	mSrcLength = 0;
	mSrcIdx = 0;
	mParserFlags = ParserFlag_None;
	mCursorIdx = -1;
	mCursorCheckIdx = -1;
	mLineStart = 0;
	//mCurToken = (BfSyntaxToken)0;
	mToken = BfToken_None;
	mSyntaxToken = BfSyntaxToken_Token;
	mTokenStart = 0;
	mTokenEnd = 0;
	mLineNum = 0;	
	mCompatMode = false;
	mQuickCompatMode = false;
	mLiteral.mWarnType = 0;
	mDataId = -1;

	mTriviaStart = 0;
	mParsingFailed = false;
	mInAsmBlock = false;
	mPreprocessorIgnoredSectionNode = NULL;
	mPreprocessorIgnoreDepth = 0;

	if (bfProject != NULL)
	{
		for (auto macro : bfProject->mPreprocessorMacros)
			mPreprocessorDefines[macro] = BfDefineState_FromProject;
	}
}


//static std::set<BfAstNode*> gFoundNodes;

BfParser::~BfParser()
{
	int parserCount = gParserCount--;

	if (mParserData->mRefCount == -1)
	{
		// Owned data, never intended for cache
		mParserData->mSrc = NULL; // Count on BfSource dtor to release strc
		delete mParserData;
	}
	else if (mParserData->mRefCount == 0)
	{
		// Just never got added to the cache
		delete mParserData;
	}
	else		
	{
		mParserData->Deref();		
	}

	mSourceData = NULL; 

	BfLogSys(mSystem, "BfParser::~BfParser %p\n", this);
}

void BfParser::SetCursorIdx(int cursorIdx)
{
	mCursorIdx = cursorIdx;
	mCursorCheckIdx = cursorIdx;

	int checkIdx = cursorIdx;
	while (checkIdx > 0)
	{
		char c = mSrc[checkIdx - 1];
		if (!IsWhitespace(c))
		{
			if (c == '.')
				mCursorCheckIdx = checkIdx;
			break;
		}
		checkIdx--;
	}
}

//static int gDeleteCount = 0;
//static int gIndentCount = 0;

void BfParser::GetLineCharAtIdx(int idx, int& line, int& lineChar)
{
	mParserData->GetLineCharAtIdx(idx, line, lineChar);
}

void BfParser::Fail(const StringImpl& error, int offset)
{
	mPassInstance->FailAt(error, mSourceData, mSrcIdx + offset);
}

void BfParser::TokenFail(const StringImpl& error, int offset)
{
	if (mPreprocessorIgnoredSectionNode == NULL)
		Fail(error, offset);
}

void BfParser::Init(uint64 cacheHash)
{
	BF_ASSERT(mParserData == NULL);
	
	mParserData = new BfParserData();
	mSourceData = mParserData;			
	mParserData->mFileName = mFileName;
	if (mDataId != -1)
		mParserData->mDataId = mDataId;
	else
		mDataId = mParserData->mDataId;
	mParserData->mAstAllocManager = &gBfParserCache->mAstAllocManager;
	mParserData->mSrc = mSrc;
	mParserData->mSrcLength = mSrcLength;
		
	if (cacheHash != 0)
	{
		BfLogSysM("Creating cached parserData %p for %p %s\n", mParserData, this, mFileName.c_str());

		mParserData->mHash = cacheHash;
		mParserData->mRefCount = 0; // 0 means we want to EVENTUALLY write it to the cache
		mSrcAllocSize = -1;
	}
	else
	{
		BfLogSysM("Creating unique parserData %p for %p %s\n", mParserData, this, mFileName.c_str());

		mParserData->mUniqueParser = this;
	}

	mJumpTableSize = ((mSrcLength + 1) + PARSER_JUMPTABLE_DIVIDE - 1) / PARSER_JUMPTABLE_DIVIDE;
	mJumpTable = new BfLineStartEntry[mJumpTableSize];
	memset(mJumpTable, 0, mJumpTableSize * sizeof(BfLineStartEntry));

	mParserData->mJumpTable = mJumpTable;
	mParserData->mJumpTableSize = mJumpTableSize;
	
	mAlloc = &mParserData->mAlloc;
	mAlloc->mSourceData = mSourceData;	
}

void BfParser::NewLine()
{
	mLineStart = mSrcIdx;
	mLineNum++;

	int idx = (mSrcIdx / PARSER_JUMPTABLE_DIVIDE);
	if (idx == 0)
		return;
	BF_ASSERT(idx < mJumpTableSize);
	BfLineStartEntry* jumpTableEntry = mJumpTable + idx;
	jumpTableEntry->mCharIdx = mSrcIdx;
	jumpTableEntry->mLineNum = mLineNum;
}

void BfParser::SetSource(const char* data, int length)
{
	const int EXTRA_BUFFER_SIZE = 80; // Extra chars for a bit of AllocChars room

	//TODO: Check cache
	// Don't cache if we have a Cursorid set, 
	//  if mDataId != -1
	//  if BfParerFlag != 0

	bool canCache = true;
	if (mDataId != -1)
		canCache = false;
	if (mParserFlags != 0)
		canCache = false;
	if (mCursorIdx != -1)
		canCache = false;
	if (mCompatMode)
		canCache = false;
	if (mQuickCompatMode)
		canCache = false;
	if (mFileName.IsEmpty())
		canCache = false;
	if (mProject == NULL)
		canCache = false;

	uint64 cacheHash = 0;
	if (canCache)
	{
		AutoCrit autoCrit(gBfParserCache->mCritSect);

		HashContext hashCtx;
		hashCtx.MixinStr(mFileName);
		hashCtx.Mixin(data, length);
		cacheHash = hashCtx.Finish64();

		BfParserCache::LookupEntry lookupEntry;
		lookupEntry.mFileName = mFileName;
		lookupEntry.mSrc = data;
		lookupEntry.mSrcLength = length;
		lookupEntry.mHash = cacheHash;
		lookupEntry.mProject = mProject;

		BfParserCache::DataEntry* dataEntryP;
		if (gBfParserCache->mEntries.TryGetWith(lookupEntry, &dataEntryP))
		{						
			mUsingCache = true;

			mParserData = dataEntryP->mParserData;
			BF_ASSERT(mParserData->mDidReduce);

			BfLogSysM("Using cached parserData %p for %p %s\n", mParserData, this, mFileName.c_str());

			mParserData->mRefCount++;
			mSourceData = mParserData;

			mSrc = mParserData->mSrc;
			mSrcLength = mParserData->mSrcLength;
			mOrigSrcLength = mParserData->mSrcLength;
			mSrcAllocSize = -1;
			mSrcIdx = 0;
			mJumpTable = mParserData->mJumpTable;
			mJumpTableSize = mParserData->mJumpTableSize;
			mAlloc = &mParserData->mAlloc;
			return;
		}
	}

	mSrcLength = length;
	mOrigSrcLength = length;
	mSrcAllocSize = mSrcLength /*+ EXTRA_BUFFER_SIZE*/;	 
	char* ownedSrc = new char[mSrcAllocSize + 1];
	memcpy(ownedSrc, data, length);
	ownedSrc[length] = 0;
	mSrc = ownedSrc;
	mSrcIdx = 0;

	Init(cacheHash);
}

void BfParser::MoveSource(const char* data, int length) // Takes ownership of data ptr
{
	mSrcLength = length;
	mOrigSrcLength = length;
	mSrcAllocSize = mSrcLength;
	mSrc = data;
	mSrcIdx = 0;
	Init();
}

void BfParser::RefSource(const char* data, int length)
{
	mSrcLength = length;
	mOrigSrcLength = length;
	mSrcAllocSize = -1;
	mSrc = data;
	mSrcIdx = 0;
	Init();
}

bool BfParser::SrcPtrHasToken(const char* name)
{
	const char* namePtr = name;
	int checkIdx = mSrcIdx - 1;
	while (*namePtr)
	{
		if (*(namePtr++) != mSrc[checkIdx])
			return false;
		checkIdx++;
	}
	if (!IsWhitespaceOrPunctuation(mSrc[checkIdx]))
		return false;
	mSrcIdx = checkIdx;
	mTokenEnd = checkIdx;
	return true;
}

void BfParser::AddErrorNode(int startIdx, int endIdx)
{
	auto identifierNode = mAlloc->Alloc<BfIdentifierNode>();
	identifierNode->Init(mTriviaStart, startIdx, endIdx);
	//identifierNode->mSource = this;
	BfSource::AddErrorNode(identifierNode);
}

BfCommentKind BfParser::GetCommentKind(int startIdx)
{
	if (((mSrc[startIdx] == '/') && (mSrc[startIdx + 1] == '*') && (mSrc[startIdx + 2] == '*') && (mSrc[startIdx + 3] == '<')) ||
		((mSrc[startIdx] == '/') && (mSrc[startIdx + 1] == '/') && (mSrc[startIdx + 2] == '/') && (mSrc[startIdx + 3] == '<')))
	{
		return BfCommentKind_Documentation_Post;
	}
	if (((mSrc[startIdx] == '/') && (mSrc[startIdx + 1] == '*') && (mSrc[startIdx + 2] == '*') && (mSrc[startIdx + 3] != '/')) ||
		((mSrc[startIdx] == '/') && (mSrc[startIdx + 1] == '/') && (mSrc[startIdx + 2] == '/') && (mSrc[startIdx + 3] != '/')))
	{
		return BfCommentKind_Documentation_Pre;
	}
	return BfCommentKind_Normal;
}

bool BfParser::EvaluatePreprocessor(BfExpression* expr)
{
	bool isInvalid = false;

	if (expr == NULL)
		return false;

	if (auto binaryOp = BfNodeDynCast<BfBinaryOperatorExpression>(expr))
	{
		switch (binaryOp->mOp)
		{
		case BfBinaryOp_ConditionalOr:
			return EvaluatePreprocessor(binaryOp->mLeft) || EvaluatePreprocessor(binaryOp->mRight);
		case BfBinaryOp_ConditionalAnd:
			return EvaluatePreprocessor(binaryOp->mLeft) && EvaluatePreprocessor(binaryOp->mRight);
		default: break;
		}
	}

	if (auto unaryOp = BfNodeDynCast<BfUnaryOperatorExpression>(expr))
	{
		switch (unaryOp->mOp)
		{
		case BfUnaryOp_Not:
			return !EvaluatePreprocessor(unaryOp->mExpression);
		default: break;
		}
	}

	if (auto identifier = BfNodeDynCast<BfIdentifierNode>(expr))
	{
		return HandleIfDef(identifier->ToString()) == MaybeBool_True;
	}

	if (auto parenExpr = BfNodeDynCast<BfParenthesizedExpression>(expr))
	{
		return EvaluatePreprocessor(parenExpr->mExpression);
	}

	if (auto literalExpr = BfNodeDynCast<BfLiteralExpression>(expr))
	{
		if (literalExpr->mValue.mTypeCode == BfTypeCode_Boolean)
		{
			return literalExpr->mValue.mBool;
		}
	}

	mPassInstance->Fail("Invalid preprocessor expression", expr);

	return false;
}

BfBlock* BfParser::ParseInlineBlock(int spaceIdx, int endIdx)
{
	BfBlock* block = NULL;
	SizedArray<BfAstNode*, 8> childArr;

	mSrcIdx = spaceIdx;
	BfAstNode* startNode = NULL;
	int usedEndIdx = spaceIdx;
	while (true)
	{
		NextToken(endIdx + 1);
		if (mSyntaxToken == BfSyntaxToken_HIT_END_IDX)
		{
			mSrcIdx = usedEndIdx;
			auto lastNode = mSidechannelRootNode->GetLast();
			if (lastNode != NULL)
				mSrcIdx = std::max(mSrcIdx, lastNode->GetSrcEnd());
			break;
		}

		usedEndIdx = mSrcIdx;

		auto childNode = CreateNode();
		if (childNode == NULL)
			break;
		if ((childNode->IsA<BfCommentNode>()))
		{
			mSidechannelRootNode->Add(childNode);			
			mPendingSideNodes.push_back(childNode);
			continue;
		}
		if (startNode == NULL)
			startNode = childNode;
		if (block == NULL)
			block = mAlloc->Alloc<BfBlock>();
		block->Add(childNode);
		childArr.push_back(childNode);
		//block->mChildArr.Add(childNode, &mAlloc);
	}

	if (block != NULL)
		block->Init(childArr, mAlloc);

	return block;
}

BfExpression* BfParser::CreateInlineExpressionFromNode(BfBlock* block)
{
	BfReducer reducer;
	reducer.mPassInstance = mPassInstance;
	reducer.mAlloc = mAlloc;
	reducer.mCompatMode = mCompatMode;
	reducer.mVisitorPos = BfReducer::BfVisitorPos(block);
	reducer.mVisitorPos.MoveNext();
	auto startNode = reducer.mVisitorPos.GetCurrent();
	if (startNode == NULL)
		return NULL;

	auto paramExpression = reducer.CreateExpression(startNode);
	if ((paramExpression != NULL) && (reducer.mVisitorPos.GetNext() != NULL))
		mPassInstance->Fail("Expression parsing error", reducer.mVisitorPos.GetNext());
	return paramExpression;
}

void BfParser::HandlePragma(const StringImpl& pragma, BfBlock* block)
{
	auto itr = block->begin();
	auto paramNode = *itr;

	if (paramNode->ToString() == "warning")
	{
		++itr;
		//auto iterNode = paramNode->mNext;		
		//BfAstNode* iterNode = parentNode->mChildArr.GetAs<BfAstNode*>(++curIdx);

		BfAstNode* iterNode = itr.Get();
		if (iterNode)
		{
			++itr;

			bool enable;
			if (iterNode->ToString() == "disable")
			{
				enable = false;
			}
			else if (iterNode->ToString() == "restore")
			{
				enable = true;
			}
			else
			{
				enable = true;
				mPassInstance->FailAt("Expected \"disable\" or \"restore\" after \"warning\"", mSourceData, iterNode->GetSrcStart(), iterNode->GetSrcLength());
			}

			//iterNode = parentNode->mChildArr.GetAs<BfAstNode*>(++curIdx);
			iterNode = itr.Get();
			while (iterNode)
			{
				++itr;
				auto tokenStr = iterNode->ToString();
				if (tokenStr != ",") // commas allowed between warning numbers but not required; we just ignore them
				{
					bool isNum = true;
					for (const auto it : tokenStr)
					{
						char c = it;
						if (c < '0' || c > '9')
						{
							isNum = false;
							break;
						}
					}
					if (isNum)
					{
						BfParserWarningEnabledChange wec;
						wec.mEnable = enable;
						wec.mWarningNumber = atoi(tokenStr.c_str());
						mParserData->mWarningEnabledChanges[iterNode->GetSrcStart()] = wec;
					}
					else
					{
						mPassInstance->FailAt("Expected decimal warning number", mSourceData, iterNode->GetSrcStart(), iterNode->GetSrcLength());
					}
				}

				//iterNode = parentNode->mChildArr.Get(++curIdx);
				iterNode = itr.Get();
			}
		}
		else
		{
			mPassInstance->FailAfterAt("Expected \"disable\" or \"restore\" after \"warning\"", mSourceData, paramNode->GetSrcEnd() - 1);
		}
	}
	else
	{
		mPassInstance->FailAt("Unknown #pragma directive", mSourceData, paramNode->GetSrcStart(), paramNode->GetSrcLength());
	}
}

void BfParser::HandleDefine(const StringImpl& name, BfAstNode* paramNode)
{	
	mPreprocessorDefines[name] = BfDefineState_ManualSet;
}

void BfParser::HandleUndefine(const StringImpl& name)
{
	mPreprocessorDefines[name] = BfDefineState_ManualUnset;	
}

MaybeBool BfParser::HandleIfDef(const StringImpl& name)
{
	BfDefineState defineState;
	if (mPreprocessorDefines.TryGetValue(name, &defineState))
	{
		if (defineState == BfDefineState_FromProject)
		{
			mParserData->mDefines_Def.Add(name);
		}

		return (defineState != BfDefineState_ManualUnset) ? MaybeBool_True : MaybeBool_False;
	}
	else
	{
		mParserData->mDefines_NoDef.Add(name);
		return MaybeBool_False;
	}
}

MaybeBool BfParser::HandleProcessorCondition(BfBlock* paramNode)
{
	if (paramNode == NULL)
		return MaybeBool_False;

	bool found = false;
	auto paramExpression = CreateInlineExpressionFromNode(paramNode);
	if (paramExpression != NULL)
	{
		return EvaluatePreprocessor(paramExpression) ? MaybeBool_True : MaybeBool_False;
	}
	return MaybeBool_False;
}

void BfParser::HandleInclude(BfAstNode* paramNode)
{

}

void BfParser::HandleIncludeNext(BfAstNode* paramNode)
{

}

void BfParser::HandlePreprocessor()
{
	int triviaStart = mTriviaStart;

	int checkIdx = 0;
	for (int checkIdx = mLineStart; checkIdx < mSrcIdx - 1; checkIdx++)
	{
		if (!isspace((uint8)mSrc[checkIdx]))
		{
			if (mPreprocessorIgnoreDepth == 0)
			{
				mPassInstance->FailAt("Preprocessor directives must appear as the first non-whitespace character on a line", mSourceData, checkIdx);
				break;
			}
			else
				continue; // Keep searching for #endif
		}
	}

	String pragma;
	String pragmaParam;

	bool atEnd = false;

	int startIdx = mSrcIdx - 1;
	int spaceIdx = -1;
	int charIdx = -1;
	while (true)
	{
		char c = mSrc[mSrcIdx++];

		if (c == '\n')
		{
			int checkIdx = mSrcIdx - 2;
			bool hadSlash = false;
			while (checkIdx >= startIdx)
			{
				char checkC = mSrc[checkIdx];
				if (checkC == '\\')
				{
					hadSlash = true;
					break;
				}
				if ((checkC != ' ') && (checkC != '\t'))
					break;
				checkIdx--;
			}

			if (!hadSlash)
				break;			
		}

		if (c == '\0')
		{
			mSrcIdx--;
			break;
		}

		if (charIdx == -1)
		{
			if (!IsWhitespaceOrPunctuation(c))
				charIdx = mSrcIdx - 1;
		}
		else if ((IsWhitespaceOrPunctuation(c)) && (spaceIdx == -1))
			spaceIdx = mSrcIdx - 1;
	}

	if (charIdx == -1)
	{
		mPassInstance->FailAt("Preprocessor directive expected", mSourceData, startIdx);
		return;
	}

	int endIdx = mSrcIdx - 1;
	while (endIdx >= startIdx)
	{
		if (!IsWhitespace(mSrc[endIdx]))
			break;
		endIdx--;
	}

	BfBlock* paramNode = NULL;
	if (spaceIdx != -1)
	{
		pragma = String(mSrc + charIdx, mSrc + spaceIdx);
		int breakIdx = spaceIdx;
		while (spaceIdx <= endIdx)
		{
			if (!isspace((uint8)mSrc[spaceIdx]))
				break;
			spaceIdx++;
		}
		if (spaceIdx <= endIdx)
			pragmaParam = String(mSrc + spaceIdx, mSrc + endIdx + 1);

		paramNode = ParseInlineBlock(breakIdx, endIdx);
	}
	else
	{
		pragma = String(mSrc + charIdx, mSrc + endIdx + 1);
		mSrcIdx = endIdx + 1;
	}

	bool wantsSingleParam = true;
	bool addToPreprocessorAccept = false;
	bool addToPreprocessorAcceptResolved = true;
	bool wantedParam = false;
	if (mPreprocessorIgnoreDepth > 0)
	{
		BF_ASSERT(!mPreprocessorNodeStack.empty());

		int ignoreEnd = std::max(mPreprocessorIgnoredSectionNode->GetSrcStart(), mLineStart - 1);
		if (pragma == "endif")
		{
			mPreprocessorIgnoreDepth--;
			if (mPreprocessorIgnoreDepth > 0)
				return;
			mPreprocessorNodeStack.pop_back();
			mPreprocessorIgnoredSectionNode->SetSrcEnd(ignoreEnd);
			mPreprocessorIgnoredSectionNode = NULL;
			triviaStart = ignoreEnd;
		}
		else if ((pragma == "if") ||
			((mCompatMode) && (pragma == "ifdef")) ||
			((mCompatMode) && (pragma == "ifndef")))
		{
			wantsSingleParam = false;
			wantedParam = true;
			mPreprocessorIgnoreDepth++;
		}
		else if (pragma == "else")
		{
			if (mCompatMode)
			{
				if (paramNode != NULL)
				{
					if (paramNode->ToString() == "if")
					{
						bool found = HandleProcessorCondition(paramNode) != MaybeBool_False;
						if (found)
						{
							addToPreprocessorAccept = true;
							mPreprocessorNodeStack.pop_back();
							mPreprocessorIgnoreDepth = 0;
							mPreprocessorIgnoredSectionNode->SetSrcEnd(ignoreEnd);
							mPreprocessorIgnoredSectionNode = NULL;
							triviaStart = ignoreEnd;
						}
						else
						{
							mPreprocessorIgnoredSectionStarts.insert(mSrcIdx);
						}
						return;
					}
				}
			}

			if ((mPreprocessorIgnoreDepth == 1) && !mPreprocessorNodeStack.back().second)
			{
				addToPreprocessorAccept = true;
				mPreprocessorNodeStack.pop_back();
				mPreprocessorIgnoreDepth = 0;
				mPreprocessorIgnoredSectionNode->SetSrcEnd(ignoreEnd);
				mPreprocessorIgnoredSectionNode = NULL;
				triviaStart = ignoreEnd;
			}
		}
		else if (pragma == "elif")
		{
			wantsSingleParam = false;
			if ((mPreprocessorIgnoreDepth == 1) && !mPreprocessorNodeStack.back().second)
			{
				wantedParam = true;
				bool found = HandleProcessorCondition(paramNode) != MaybeBool_False;

				if (found)
				{
					addToPreprocessorAccept = true;
					mPreprocessorNodeStack.pop_back();
					mPreprocessorIgnoreDepth = 0;
					mPreprocessorIgnoredSectionNode->SetSrcEnd(ignoreEnd);
					mPreprocessorIgnoredSectionNode = NULL;
					triviaStart = ignoreEnd;
				}
				else
				{
					mPreprocessorIgnoredSectionStarts.insert(mSrcIdx);
				}
			}
		}

		if (mPreprocessorIgnoreDepth > 0)
			return;
	}
	else
	{
		if ((pragma == "if") ||
			((mCompatMode) && (pragma == "ifdef")) ||
			((mCompatMode) && (pragma == "ifndef")))
		{
			wantsSingleParam = false;
			wantedParam = true;
			bool found = false;
			if (!mQuickCompatMode)
			{
				if (pragma == "if")
					found = HandleProcessorCondition(paramNode) != MaybeBool_False;
				else if (pragma == "ifdef")
					found = HandleIfDef(pragmaParam) != MaybeBool_False;
				else if (pragma == "ifndef")
					found = HandleIfDef(pragmaParam) != MaybeBool_True;
			}

			if (!found)
				mPreprocessorIgnoredSectionStarts.insert(mSrcIdx);

			addToPreprocessorAccept = true;
			if ((!found) && (!mQuickCompatMode) && (!mCompleteParse))
			{
				addToPreprocessorAcceptResolved = false;
				mPreprocessorIgnoreDepth = 1;
			}
		}
		else if (pragma == "else")
		{
			if (!mQuickCompatMode && !mCompleteParse)
			{
				if (mPreprocessorNodeStack.empty())
					mPassInstance->FailAt("Unexpected #else", mSourceData, startIdx, mSrcIdx - startIdx);
				else
				{
					BF_ASSERT(mPreprocessorNodeStack.back().second);
					mPreprocessorIgnoreDepth = 1;
				}
			}
		}
		else if (pragma == "elif")
		{
			wantsSingleParam = false;
			if (!mQuickCompatMode && !mCompleteParse)
			{
				if (mPreprocessorNodeStack.empty())
					mPassInstance->FailAt("Unexpected #elif", mSourceData, startIdx, mSrcIdx - startIdx);
				else
				{
					BF_ASSERT(mPreprocessorNodeStack.back().second);
					mPreprocessorIgnoreDepth = 1;
				}
			}
			wantedParam = true;
		}
		else if (pragma == "endif")
		{
			if (mPreprocessorNodeStack.empty())
				mPassInstance->FailAt("Unexpected #endif", mSourceData, startIdx, mSrcIdx - startIdx);
			else
				mPreprocessorNodeStack.pop_back();
		}
		else if (pragma == "define")
		{
			HandleDefine(pragmaParam, paramNode);
			wantedParam = true;
		}
		else if (pragma == "undef")
		{
			HandleUndefine(pragmaParam);
			wantedParam = true;
		}
		else if (pragma == "error")
		{
			if (!mQuickCompatMode)
			{
				//TODO: Remove
				//OutputDebugStrF("####################ERROR %s\n", pragmaParam.c_str());
			}

			mPassInstance->FailAt(pragmaParam, mSourceData, startIdx);
			wantedParam = true;
		}
		else if (pragma == "warning")
		{
			wantsSingleParam = false;
			mPassInstance->WarnAt(BfWarning_CS1030_PragmaWarning, pragmaParam, mSourceData, startIdx);
			wantedParam = true;
		}
		else if (pragma == "region")
		{
			wantsSingleParam = false;
			wantedParam = true;
		}
		else if (pragma == "endregion")
		{
			wantsSingleParam = false;
			if (!pragmaParam.empty())
				wantedParam = true;
		}
		else if (pragma == "pragma")
		{
			wantsSingleParam = false;
			wantedParam = true;
			if (paramNode != NULL)
				HandlePragma(pragmaParam, paramNode);
		}
		else if (pragma == "unwarn")
		{
			mParserData->mUnwarns.insert(mSrcIdx);
		}
		else if ((mCompatMode) && (pragma == "include"))
		{
			HandleInclude(paramNode);
			wantedParam = true;
		}
		else if ((mCompatMode) && (pragma == "include_next"))
		{
			HandleIncludeNext(paramNode);
			wantedParam = true;
		}
		else
		{
			mPassInstance->FailAt("Unknown preprocessor directive", mSourceData, startIdx, mSrcIdx - startIdx);
		}
	}

	if ((wantsSingleParam) && (paramNode != NULL) && (paramNode->mChildArr.size() > 1))
	{
		mPassInstance->FailAt("Only one parameter expected", mSourceData, paramNode->GetSrcStart(), paramNode->GetSrcLength());
	}

	if ((wantedParam) && (paramNode == NULL))
	{
		mPassInstance->FailAt("Expected parameter", mSourceData, startIdx, mSrcIdx - startIdx);
	}
	else if ((!wantedParam) && (paramNode != NULL))
	{
		mPassInstance->FailAt("Parameter not expected", mSourceData, startIdx, mSrcIdx - startIdx);
	}

	mTokenStart = charIdx;
	mTokenEnd = charIdx + (int)pragma.length();
	mTriviaStart = -1;
	auto bfPreprocessorCmdNode = mAlloc->Alloc<BfIdentifierNode>();
	bfPreprocessorCmdNode->Init(this);

	mTriviaStart = triviaStart;
	auto bfPreprocessorNode = mAlloc->Alloc<BfPreprocessorNode>();
	mTokenStart = startIdx;
	mTokenEnd = mSrcIdx;
	bfPreprocessorNode->Init(this);
	bfPreprocessorNode->Add(bfPreprocessorCmdNode);
	bfPreprocessorNode->mCommand = bfPreprocessorCmdNode;

	if (paramNode != NULL)
	{
		int curIdx = 0;
		bfPreprocessorNode->mArgument = paramNode;
	}
	mPendingSideNodes.push_back(bfPreprocessorNode);
	mTokenStart = mSrcIdx;
	mTriviaStart = mSrcIdx;
	triviaStart = mSrcIdx;

	if (addToPreprocessorAccept)
		mPreprocessorNodeStack.push_back(std::pair<BfAstNode*, bool>(bfPreprocessorNode, addToPreprocessorAcceptResolved));

	if (mPreprocessorIgnoreDepth > 0)
	{
		mPreprocessorIgnoredSectionNode = mAlloc->Alloc<BfPreprocesorIgnoredSectionNode>();
		mPreprocessorIgnoredSectionNode->Init(this);
		mSidechannelRootNode->Add(mPreprocessorIgnoredSectionNode);
		mPendingSideNodes.push_back(mPreprocessorIgnoredSectionNode);
	}
}

static int ValSign(int64 val)
{
	if (val < 0)
		return -1;
	if (val > 0)
		return 1;
	return 0;
}

template <int Len>
struct StrHashT
{
	const static int HASH = 0;
};

template <>
struct StrHashT<4>
{
	template <const char* Str>
	struct DoHash
	{
		const static int HASH = (StrHashT<3>::HASH) ^ Str[4];
	};

};

// This is little endian only
#define TOKEN_HASH(a, b, c, d) ((int)a << 0) | ((int)b << 8) | ((int)c << 16) | ((int)d << 24)

const int text_const = (1 << 2);
const int gClassConst = 0;

uint32 BfParser::GetTokenHash()
{
	char hashChars[4] = { 0 };
	int idx = 0;
	uint32 tokenHash = 0;
	int checkIdx = mSrcIdx - 1;
	while ((!IsWhitespaceOrPunctuation(mSrc[checkIdx])) && (idx < 4))
	{
		hashChars[idx++] = mSrc[checkIdx];
		checkIdx++;
	}
	return *((uint32*)hashChars);
}

double BfParser::ParseLiteralDouble()
{
	char buf[256];
	int len = std::min(mTokenEnd - mTokenStart, 255);

	memcpy(buf, &mSrc[mTokenStart], len);
	char c = buf[len - 1];
	if ((c == 'd') || (c == 'D') || (c == 'f') || (c == 'F'))
		buf[len - 1] = '\0';
	else
		buf[len] = '\0';

	return strtod(buf, NULL);
}

void BfParser::NextToken(int endIdx)
{
	mToken = BfToken_None;
	if (mSyntaxToken == BfSyntaxToken_EOF)
		Fail("Unexpected end of file");

	mTriviaStart = mSrcIdx;

	bool isLineStart = true;
	bool isVerbatim = false;
	int verbatimStart = -1;

	while (true)
	{
		bool setVerbatim = false;
		uint32 checkTokenHash = 0;

		if ((endIdx != -1) && (mSrcIdx >= endIdx))
		{
			mSyntaxToken = BfSyntaxToken_HIT_END_IDX;
			return;
		}

		mTokenStart = mSrcIdx;
		mTokenEnd = mSrcIdx + 1;
		char c = mSrc[mSrcIdx++];

		if ((mPreprocessorIgnoreDepth > 0) && (endIdx == -1))
		{
			if (c == 0)
			{
				mSyntaxToken = BfSyntaxToken_EOF;
				mSrcIdx--;
				break;
			}

			if ((c != '#') || (!isLineStart))
			{
				if (c == '\n')
				{
					NewLine();
					isLineStart = true;
					continue;
				}

				if (IsWhitespace(c))
					continue;

				isLineStart = false;
				continue;
			}
		}

		switch (c)
		{
		case '!':
			if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_CompareNotEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Bang;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '=':
			if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_CompareEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else if (mSrc[mSrcIdx] == '>')
			{
				mToken = BfToken_FatArrow;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_AssignEquals;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '+':
			if (mSrc[mSrcIdx] == '+')
			{
				mToken = BfToken_DblPlus;
				mTokenEnd = ++mSrcIdx;
			}
			else if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_PlusEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Plus;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '^':
			if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_XorEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Carat;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '~':
			mToken = BfToken_Tilde;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '%':
			if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_ModulusEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Modulus;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '&':
			if (mSrc[mSrcIdx] == '&')
			{
				mToken = BfToken_DblAmpersand;
				mTokenEnd = ++mSrcIdx;
			}
			else if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_AndEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Ampersand;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '|':
			if (mSrc[mSrcIdx] == '|')
			{
				mToken = BfToken_DblBar;
				mTokenEnd = ++mSrcIdx;
			}
			else if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_OrEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Bar;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '*':
			if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_MultiplyEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Star;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '?':
			if (mSrc[mSrcIdx] == '?')
			{
				mToken = BfToken_DblQuestion;
				mTokenEnd = ++mSrcIdx;
			}
			else if (mSrc[mSrcIdx] == '.')
			{
				mToken = BfToken_QuestionDot;
				mTokenEnd = ++mSrcIdx;
			}
			else if (mSrc[mSrcIdx] == '[')
			{
				mToken = BfToken_QuestionLBracket;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_Question;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '<':
			if (mSrc[mSrcIdx] == '<')
			{
				mTokenEnd = ++mSrcIdx;
				if (mSrc[mSrcIdx] == '=')
				{
					mToken = BfToken_ShiftLeftEquals;
					mTokenEnd = ++mSrcIdx;
				}
				else
					mToken = BfToken_LDblChevron;
			}
			else if (mSrc[mSrcIdx] == '=')
			{
				if (mSrc[mSrcIdx + 1] == '>')
				{
					mToken = BfToken_Spaceship;
					mSrcIdx += 2;
					mTokenEnd = mSrcIdx;
				}
				else
				{
					mToken = BfToken_LessEquals;
					mTokenEnd = ++mSrcIdx;
				}
			}
			else
				mToken = BfToken_LChevron;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '>':
			if (mSrc[mSrcIdx] == '>')
			{
				mTokenEnd = ++mSrcIdx;
				if (mSrc[mSrcIdx] == '=')
				{
					mToken = BfToken_ShiftRightEquals;
					mTokenEnd = ++mSrcIdx;
				}
				else
					mToken = BfToken_RDblChevron;
			}
			else if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_GreaterEquals;
				mTokenEnd = ++mSrcIdx;
			}
			else
				mToken = BfToken_RChevron;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '@':
			setVerbatim = true;
			c = mSrc[mSrcIdx];
			if (c == '\"')
			{
				setVerbatim = true;
			}
			else if (((c >= 'A') && (c <= 'a')) || ((c >= 'a') && (c <= 'z')) || (c == '_') || (c == '@'))
			{
				setVerbatim = true;
			}
			else
			{
				mSyntaxToken = BfSyntaxToken_Identifier;
				//mToken = BfToken_At;
				//mSyntaxToken = BfSyntaxToken_Token;
				//Fail("Keyword, identifier, or string expected after verbatim specifier: @"); // CS1646
				return;
			}
			break;
		case '"':
		case '\'':
		{
			String lineHeader;
			String strLiteral;
			char startChar = c;
			bool isMultiline = false;

			if ((mSrc[mSrcIdx] == '"') && (mSrc[mSrcIdx + 1] == '"'))
			{
				isMultiline = true;
				mSrcIdx += 2;
			}

			int contentErrorStart = -1;
			int lineIdx = 0;
			int lineIndentIdx = -1;
			if (isMultiline)
			{
				int checkIdx = mSrcIdx;
				int lineStartIdx = checkIdx;

				while (true)
				{
					char c = mSrc[checkIdx++];

					if ((c == '"') && (mSrc[checkIdx] == '"') && (mSrc[checkIdx + 1] == '"'))
					{
						lineIndentIdx = lineStartIdx;
						for (int i = lineStartIdx; i < checkIdx - 1; i++)
						{
							char c = mSrc[i];
							if ((c != '\t') && (c != ' '))
							{
								mPassInstance->FailAt("Multi-line string literal closing delimiter must begin on a new line", mSourceData, i, checkIdx - i + 3);
								break;
							}

							lineHeader.Append(c);
						}
						break;
					}
					else if (c == '\n')
					{
						if (contentErrorStart != -1)
						{
							mPassInstance->FailAt("Multi-line string literal content must begin on a new line", mSourceData, contentErrorStart, checkIdx - contentErrorStart - 1);
							contentErrorStart = -1;
						}

						lineStartIdx = checkIdx;
						lineIdx++;
					}
					else if (c == '\0')
						break; // Will throw an error in next pass
					else if ((c == ' ') || (c == '\t') || (c == '\r'))
					{
						// Allow
					}
					else if (lineIdx == 0)
					{
						if (contentErrorStart == -1)
							contentErrorStart = checkIdx - 1;
					}
				}
			}

			int lineCount = lineIdx + 1;
			lineIdx = 0;
			int lineStart = mSrcIdx;

			while (true)
			{
				char c = mSrc[mSrcIdx++];
				if (c == '\0')
				{
					// Invalid file end
					mPassInstance->FailAt("String not terminated", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
					mSrcIdx--;
					break;
				}
				else if (c == '\n')
				{
					if (isMultiline)
					{
						lineIdx++;

						if ((lineIdx > 1) && (lineIdx < lineCount - 1))
						{
							strLiteral += "\n";
						}

						lineStart = mSrcIdx;
						for (int i = 0; i < lineHeader.GetLength(); i++)
						{
							char wantC = lineHeader[i];
							char c = mSrc[mSrcIdx];
							if (c == '\r')
								continue;
							if (c == '\n')
								break;
							if (wantC == c)
							{
								mSrcIdx++;
							}
							else
							{
								BfError* error = NULL;
								if (c == ' ')
								{
									error = mPassInstance->FailAt("Unexpected space in indentation of line in multi-line string literal", mSourceData, mSrcIdx, 1, BfFailFlag_ShowSpaceChars);
								}
								else if (c == '\t')
								{
									error = mPassInstance->FailAt("Unexpected tab in indentation of line in multi-line string literal", mSourceData, mSrcIdx, 1, BfFailFlag_ShowSpaceChars);
								}
								else
								{
									error = mPassInstance->FailAt("Insufficient indentation of line in multi-line string literal", mSourceData, lineStart, mSrcIdx - lineStart + 1, BfFailFlag_ShowSpaceChars);
								}
								if (error != NULL)
								{
									mPassInstance->MoreInfoAt("Change indentation of this line to match closing delimiter", mSourceData, lineIndentIdx, lineHeader.GetLength(), BfFailFlag_ShowSpaceChars);
								}
								break;
							}
						}

						NewLine();
					}
					else
					{
						mSrcIdx--;
						int errorIdx = mSrcIdx - 1;
						while ((errorIdx > 0) && (IsWhitespace(mSrc[errorIdx])))
							errorIdx--;
						mPassInstance->FailAfterAt("Newline not allowed in string", mSourceData, errorIdx);
						break;
					}
				}
				else if ((c == '"') && (c == startChar))
				{
					if (isMultiline)
					{
						if ((mSrc[mSrcIdx] == '"') && (mSrc[mSrcIdx + 1] == '"')) // Triple quote
						{
							// Done
							mSrcIdx += 2;
							break;
						}

						strLiteral += '"';
					}
					else
					{
						if (mSrc[mSrcIdx] == '"') // Double quote
						{
							strLiteral += '"';
							mSrcIdx++;
						}
						else
							break;
					}
				}
				else if ((c == '\'') && (c == startChar))
				{
					break;
				}
				else if ((c == '\\') && (!isVerbatim))
				{
					char c = mSrc[mSrcIdx++];
					switch (c)
					{
					case '0':
						strLiteral += '\0';
						break;
					case 'a':
						strLiteral += '\a';
						break;
					case 'b':
						strLiteral += '\b';
						break;
					case 'f':
						strLiteral += '\f';
						break;
					case 'n':
						strLiteral += '\n';
						break;
					case 'r':
						strLiteral += '\r';
						break;
					case 't':
						strLiteral += '\t';
						break;
					case 'v':
						strLiteral += '\v';
						break;
					case '\\':
					case '"':
					case '\'':
						strLiteral += c;
						break;
					case 'x':
					{
						int wantHexChars = 2;

						int hexVal = 0;
						int numHexChars = 0;
						while (true)
						{
							char c = mSrc[mSrcIdx];
							int hexChar = 0;

							if ((c >= '0') && (c <= '9'))
								hexChar = c - '0';
							else if ((c >= 'a') && (c <= 'f'))
								hexChar = c - 'a' + 0xa;
							else if ((c >= 'A') && (c <= 'F'))
								hexChar = c - 'A' + 0xA;
							else
							{
								Fail("Expected two hex characters");
								break;
							}
							mSrcIdx++;
							numHexChars++;
							hexVal = (hexVal * 0x10) + hexChar;

							if (numHexChars == wantHexChars)
								break;
						}

						strLiteral += (char)hexVal;
					}
					break;
					case 'u':
					{
						if (mSrc[mSrcIdx] != '{')
						{
							Fail("Expected hexadecimal code in braces after unicode escape");
							break;
						}
						mSrcIdx++;

						int hexStart = mSrcIdx;

						int hexVal = 0;
						int numHexChars = 0;
						while (true)
						{
							char c = mSrc[mSrcIdx];
							int hexChar = 0;

							if (c == '}')
							{
								if (numHexChars == 0)
									Fail("Unicode escape sequence expects hex digits");

								mSrcIdx++;
								break;
							}

							if ((c >= '0') && (c <= '9'))
								hexChar = c - '0';
							else if ((c >= 'a') && (c <= 'f'))
								hexChar = c - 'a' + 0xa;
							else if ((c >= 'A') && (c <= 'F'))
								hexChar = c - 'A' + 0xA;
							else
							{
								Fail("Hex encoding error");
								break;
							}
							mSrcIdx++;
							numHexChars++;

							if (numHexChars > 8)
							{
								Fail("Too many hex digits for an unicode scalar");
							}

							hexVal = (hexVal * 0x10) + hexChar;
						}

						char outStrUTF8[8];
						int size = u8_toutf8(outStrUTF8, 8, (uint32)hexVal);
						if (size == 0)
						{
							mPassInstance->FailAt("Invalid unicode scalar", mSourceData, hexStart, mSrcIdx - hexStart - 1);
						}
						strLiteral += outStrUTF8;
					}
					break;
					default:
						Fail("Unrecognized escape sequence");
						strLiteral += c;
					}
				}
				else
					strLiteral += c;
			}

			if (isVerbatim)
			{
				mTokenStart--;
			}

			mTokenEnd = mSrcIdx;
			mSyntaxToken = BfSyntaxToken_Literal;
			if (startChar == '\'')
			{
				mLiteral.mTypeCode = BfTypeCode_Char8;
				if (strLiteral.length() == 0)
				{
					if (mPreprocessorIgnoredSectionNode == NULL)
						mPassInstance->FailAt("Empty char literal", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
				}
				else if (strLiteral.length() > 1)
				{
					int utf8Len = u8_seqlen((char*)strLiteral.c_str());
					if (utf8Len == (int)strLiteral.length())
					{
						mLiteral.mUInt64 = u8_toucs((char*)strLiteral.c_str(), (int)strLiteral.length());
					}
					else if (mPreprocessorIgnoredSectionNode == NULL)
					{
						bool isGraphemeCluster = false;

						// There's no explicit unicode limit to how many diacriticals a grapheme cluster can contain, 
						// but we apply a limit for sanity for the purpose of this error
						if (strLiteral.length() < 64)
						{
							int numCodePoints;
							int numCombiningMarks;
							UTF8Categorize(strLiteral.c_str(), (int)strLiteral.length(), numCodePoints, numCombiningMarks);
							isGraphemeCluster = numCodePoints - numCombiningMarks <= 1;
						}

						if (isGraphemeCluster)
							mPassInstance->FailAt("Grapheme clusters cannot be used as character literals", mSourceData, mTokenStart + 1, mSrcIdx - mTokenStart - 2);
						else
							mPassInstance->FailAt("Too many characters in character literal", mSourceData, mTokenStart + 1, mSrcIdx - mTokenStart - 2);
					}
				}
				else
				{
					mLiteral.mInt64 = (uint8)strLiteral[0];
				}

				if (mLiteral.mInt64 >= 0x10000)
					mLiteral.mTypeCode = BfTypeCode_Char32;
				else if (mLiteral.mInt64 >= 0x100)
					mLiteral.mTypeCode = BfTypeCode_Char16;
			}
			else
			{
				auto* strLiteralPtr = new String(std::move(strLiteral));
				mParserData->mStringLiterals.push_back(strLiteralPtr);
				mLiteral.mTypeCode = BfTypeCode_CharPtr;
				mLiteral.mString = strLiteralPtr;
			}
			return;
		}
		break;
		case '/':
			if (mSrc[mSrcIdx] == '/')
			{
				// Comment line
				while (true)
				{
					char c = mSrc[mSrcIdx++];
					if ((c == '\n') || (c == '\0'))
					{
						mSrcIdx--;
						break;
					}
				}
				mTokenEnd = mSrcIdx;

				if (mPreprocessorIgnoredSectionNode == NULL)
				{
					bool handled = false;
					if (!mPendingSideNodes.IsEmpty())
					{
						if (auto prevComment = BfNodeDynCast<BfCommentNode>(mPendingSideNodes.back()))
						{							
							// This is required for folding '///' style multi-line documentation into a single node
							if (prevComment->GetTriviaStart() == mTriviaStart)
							{
								if (GetCommentKind(prevComment->mSrcStart) == GetCommentKind(mTokenStart))
								{
									prevComment->SetSrcEnd(mSrcIdx);
									handled = true;
								}
							}
						}
					}

					if (!handled)
					{
						auto bfCommentNode = mAlloc->Alloc<BfCommentNode>();
						bfCommentNode->Init(this);						
						bfCommentNode->mCommentKind = GetCommentKind(mTokenStart);
						mSidechannelRootNode->Add(bfCommentNode);
						mPendingSideNodes.push_back(bfCommentNode);
					}
				}
				break;
			}
			else if (mSrc[mSrcIdx] == '=')
			{
				mToken = BfToken_DivideEquals;
				mTokenEnd = ++mSrcIdx;
				mSyntaxToken = BfSyntaxToken_Token;
				return;
			}
			else if (mSrc[mSrcIdx] == '*')
			{
				// Comment block
				int nestCount = 1;
				mSrcIdx++;
				while (true)
				{
					char c = mSrc[mSrcIdx++];
					if (c == '\n')
					{
						NewLine();
					}
					else if ((c == '\0') || ((c == '*') && (mSrc[mSrcIdx] == '/')))
					{
						// Block ends
						if (c == '\0')
						{
							nestCount = 0;
							mSrcIdx--;
						}
						else
						{
							c = 0;
							nestCount--;
							mSrcIdx++;
						}

						if (nestCount == 0)
						{
							mTokenEnd = mSrcIdx;
							if (mPreprocessorIgnoredSectionNode == NULL)
							{
								bool handled = false;
								if (!mPendingSideNodes.IsEmpty())
								{
									if (auto prevComment = BfNodeDynCast<BfCommentNode>(mPendingSideNodes.back()))
									{
										// This is required for folding documentation into a single node
										if (prevComment->GetTriviaStart() == mTriviaStart)
										{
											if (GetCommentKind(prevComment->mSrcStart) == GetCommentKind(mTokenStart))
											{
												prevComment->SetSrcEnd(mSrcIdx);
												handled = true;
											}
										}
									}
								}

								if (!handled)
								{
									auto bfCommentNode = mAlloc->Alloc<BfCommentNode>();
									bfCommentNode->Init(this);
									bfCommentNode->mCommentKind = GetCommentKind(mTokenStart);
									mSidechannelRootNode->Add(bfCommentNode);
									mPendingSideNodes.push_back(bfCommentNode);
								}
							}
							break;
						}
					}
					else if ((!mCompatMode) && ((c == '/') && (mSrc[mSrcIdx] == '*') && (mSrc[mSrcIdx - 2] != '/')))
					{
						nestCount++;
						mSrcIdx++;
					}
				}
			}
			else
			{
				mToken = BfToken_ForwardSlash;
				mSyntaxToken = BfSyntaxToken_Token;
				return;
			}
			break;
		case '#':
			HandlePreprocessor();
			if (mSyntaxToken == BfSyntaxToken_EOF)
				return;
			break;
		case '.':
			if (mSrc[mSrcIdx] == '.')
			{
				if (mSrc[mSrcIdx + 1] == '.')
				{
					mSrcIdx += 2;
					mTokenEnd = mSrcIdx;
					mToken = BfToken_DotDotDot;
					mSyntaxToken = BfSyntaxToken_Token;
				}
				else
				{
					mSrcIdx++;
					mTokenEnd = mSrcIdx;
					mToken = BfToken_DotDot;
					mSyntaxToken = BfSyntaxToken_Token;
				}
			}
			else
			{
				mToken = BfToken_Dot;
				mSyntaxToken = BfSyntaxToken_Token;
			}
			return;
		case ',':
			mToken = BfToken_Comma;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case ';':
			if (mInAsmBlock)
			{
				mToken = BfToken_AsmNewline;
				mSyntaxToken = BfSyntaxToken_Token;
			}
			else
			{
				mToken = BfToken_Semicolon;
				mSyntaxToken = BfSyntaxToken_Token;
			}
			return;
		case ':':
		{
			if ((mCompatMode) && (mSrc[mSrcIdx] == ':'))
			{
				mSrcIdx++;
				mTokenEnd = mSrcIdx;
				mToken = BfToken_Dot;
				mSyntaxToken = BfSyntaxToken_Token;
			}
			else
			{
				mToken = BfToken_Colon;
				mSyntaxToken = BfSyntaxToken_Token;
			}
		}
		return;
		case '(':
			mToken = BfToken_LParen;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case ')':
			mToken = BfToken_RParen;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '{':
			mToken = BfToken_LBrace;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '}':
			mToken = BfToken_RBrace;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '[':
			mToken = BfToken_LBracket;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case ']':
			mToken = BfToken_RBracket;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case '\n':
			NewLine();
			if (!mInAsmBlock)
				continue;
			mToken = BfToken_AsmNewline;
			mSyntaxToken = BfSyntaxToken_Token;
			return;
		case ' ':
		case '\t':
		case '\v':
		case '\f':
		case '\r':
			continue; // Whitespace
		case '\0':
			mSrcIdx--; // Stay on EOF marker
			mSyntaxToken = BfSyntaxToken_EOF;
			return;
		default:
			if (((c >= '0') && (c <= '9')) || (c == '-'))
			{				
				bool prevIsDot = (mSrcIdx > 1) && (mSrc[mSrcIdx - 2] == '.');

				if (c == '-')
				{
					// Not a number!
					if (mSrc[mSrcIdx] == '-')
					{
						mToken = BfToken_DblMinus;
						mSrcIdx++;
					}
					else if (mSrc[mSrcIdx] == '=')
					{
						mToken = BfToken_MinusEquals;
						mSrcIdx++;
					}
					else if ((mCompatMode) && (mSrc[mSrcIdx] == '>'))
					{
						mToken = BfToken_Dot;
						mSrcIdx++;
					}
					else
						mToken = BfToken_Minus;
					mSyntaxToken = BfSyntaxToken_Token;
					mTokenEnd = mSrcIdx;
					return;
				}

				bool wasNeg = false;
				bool hadOverflow = false;
				int64 val = 0;
				int numberBase = 10;
				int expVal = 0;
				int expSign = 0;
				bool hasExp = false;
				bool hadSeps = false;
				bool hadLeadingHexSep = false;
				int hexDigits = 0;
				if (c == '-')
				{
					wasNeg = true; //TODO: This never actually gets set any more (eaten as BfToken_Minus above).  Move checks that use this to later in pipeline, then remove this
					c = mSrc[mSrcIdx++];
				}

				val = c - '0';

				if (c == '0')
				{
					switch (mSrc[mSrcIdx])
					{
					case 'b':
					case 'B':
						numberBase = 2;
						mSrcIdx++;
						break;
					case 'o':
					case 'O':
						numberBase = 8;
						mSrcIdx++;
						break;
					case 'x':
					case 'X':
						numberBase = 16;
						mSrcIdx++;
						break;
					}
				}

				while (true)
				{
					char c = mSrc[mSrcIdx++];

					if (c == '\'')
					{
						hadSeps = true;
						if ((numberBase == 0x10) && (hexDigits == 0))
							hadLeadingHexSep = true;
						continue;
					}

					if ((numberBase == 10) && ((c == 'e') || (c == 'E')))
					{
						// Specifying exponent

						while (true)
						{
							c = mSrc[mSrcIdx++];
							if (c == '+')
							{
								if (expSign != 0)
									TokenFail("Format error");
								expSign = 1;
							}
							else if (c == '-')
							{
								if (expSign != 0)
									TokenFail("Format error");
								expSign = -1;
							}
							else if ((c >= '0') && (c <= '9'))
							{
								hasExp = true;
								expVal *= 10;
								expVal += c - '0';
							}
							else
							{
								if (expSign == -1)
									expVal = -expVal;
								mSrcIdx--;
								break;
							}
						}
						if (!hasExp)
						{
							TokenFail("Expected an exponent");
						}
					}

					// The 'prevIsDot' helps tuple lookups like "tuple.0.0", interpreting those as two integers rather than a float
					if (((c == '.') && (!prevIsDot)) || (hasExp))
					{
						// Switch to floating point mode
						//double dVal = val;
						//double dValScale = 0.1; 
						//if (hasExp)
						//dVal *= pow(10, expVal);
						while (true)
						{
							char c = mSrc[mSrcIdx++];

							if (IsWhitespaceOrPunctuation(c))
							{
								mTokenEnd = mSrcIdx - 1;
								mSrcIdx--;
								mLiteral.mTypeCode = BfTypeCode_Double;
								mLiteral.mDouble = ParseLiteralDouble();//dVal;
								mSyntaxToken = BfSyntaxToken_Literal;
								return;
							}

							if ((c == 'e') || (c == 'E'))
							{
								// Specifying exponent

								if (hasExp)
									TokenFail("Format error");

								while (true)
								{
									c = mSrc[mSrcIdx++];
									if (c == '+')
									{
										if (expSign != 0)
											TokenFail("Format error");
										expSign = 1;
									}
									else if (c == '-')
									{
										if (expSign != 0)
											TokenFail("Format error");
										expSign = -1;
									}
									else if ((c >= '0') && (c <= '9'))
									{
										hasExp = true;
										expVal *= 10;
										expVal += c - '0';
									}
									else
									{
										if (expSign == -1)
											expVal = -expVal;
										mSrcIdx--;
										//dVal *= pow(10, expVal);
										break;
									}
								}
								if (!hasExp)
								{
									TokenFail("Expected an exponent");
								}
								continue;
							}

							if ((c == 'f') || (c == 'F'))
							{
								mTokenEnd = mSrcIdx;
								mLiteral.mTypeCode = BfTypeCode_Single;
								mLiteral.mSingle = (float)ParseLiteralDouble();//(float)dVal;
								mSyntaxToken = BfSyntaxToken_Literal;
								return;
							}
							else if ((c == 'd') || (c == 'D'))
							{
								mTokenEnd = mSrcIdx;
								mLiteral.mTypeCode = BfTypeCode_Double;
								mLiteral.mDouble = ParseLiteralDouble();//(double)dVal;
								mSyntaxToken = BfSyntaxToken_Literal;
								return;
							}
							else if ((c >= '0') && (c <= '9'))
							{
								//dVal += (c - '0') * dValScale;
								//dValScale *= 0.1;
							}
							else if ((((c >= 'A') && (c <= 'Z')) || ((c >= 'a') && (c <= 'z'))) &&
								(mSrc[mSrcIdx - 2] == '.'))
							{
								// This is actually a integer followed by an Int32 call (like 123.ToString)
								mSrcIdx -= 2;
								mTokenEnd = mSrcIdx;
								mLiteral.mInt64 = val;
								mLiteral.mTypeCode = BfTypeCode_IntUnknown;
								mSyntaxToken = BfSyntaxToken_Literal;
								return;
							}
							else
							{
								mSrcIdx--;
								mTokenEnd = mSrcIdx;
								mLiteral.mTypeCode = BfTypeCode_Double;
								mLiteral.mDouble = ParseLiteralDouble();//(double)dVal;
								mSyntaxToken = BfSyntaxToken_Literal;
								TokenFail("Unexpected character while parsing number", 0);
								return;
							}
						}
						return;
					}

					if (IsWhitespaceOrPunctuation(c))
					{
						mTokenEnd = mSrcIdx - 1;
						mSrcIdx--;
						if (wasNeg)
							val = -val;

						if ((numberBase == 0x10) &&
							((hexDigits == 16) || ((hadSeps) && (hexDigits > 8)) || ((hadLeadingHexSep) && (hexDigits == 8))))
						{
							if (hexDigits > 16)
								mPassInstance->FailAt("Too many hex digits for int64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
							mLiteral.mInt64 = val;
							if ((val < 0) && (!wasNeg))
								mLiteral.mTypeCode = BfTypeCode_UInt64;
							else
								mLiteral.mTypeCode = BfTypeCode_Int64;
						}
						else
						{
							mLiteral.mInt64 = val;
							mLiteral.mTypeCode = BfTypeCode_IntUnknown;

							if ((hadOverflow) || (val < -0x80000000LL) || (val > 0xFFFFFFFFLL))
								mPassInstance->FailAt("Value doesn't fit into int32", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
							else
							{
								if ((numberBase == 0x10) && (hexDigits == 7))
									mLiteral.mWarnType = BfWarning_BF4201_Only7Hex;
								//mPassInstance->WarnAt(0, "Only 7 hex digits specified.  Add a leading zero to clarify intention.", this, mTokenStart, mSrcIdx - mTokenStart);
								if ((numberBase == 0x10) && (hexDigits > 8))
									mLiteral.mWarnType = BfWarning_BF4202_TooManyHexForInt;
								//mPassInstance->FailAt("Too many hex digits for an int, but too few for a long.  Use 'L' suffix if a long was intended.", this, mTokenStart, mSrcIdx - mTokenStart);
								/*if (val > 0x7FFFFFFF)
								mLiteral.mTypeCode = BfTypeCode_UIntUnknown;*/
							}
						}

						mSyntaxToken = BfSyntaxToken_Literal;
						return;
					}

					int64 prevVal = val;
					if ((c >= '0') && (c <= '9') && (c < '0' + numberBase))
					{
						if (numberBase == 0x10)
							hexDigits++;
						val *= numberBase;
						val += c - '0';
					}
					else if ((numberBase == 0x10) && (c >= 'A') && (c <= 'F'))
					{
						hexDigits++;
						val *= numberBase;
						val += c - 'A' + 0xA;
					}
					else if ((numberBase == 0x10) && (c >= 'a') && (c <= 'f'))
					{
						hexDigits++;
						val *= numberBase;
						val += c - 'a' + 0xa;
					}
					
					else if ((c == 'u') || (c == 'U'))
					{
						if (wasNeg)
							val = -val;
						if ((mSrc[mSrcIdx] == 'l') || (mSrc[mSrcIdx] == 'L'))
						{
							if (mSrc[mSrcIdx] == 'l')
								TokenFail("Uppercase 'L' required for int64");
							mSrcIdx++;
							mTokenEnd = mSrcIdx;
							mLiteral.mTypeCode = BfTypeCode_UInt64;
							mLiteral.mUInt64 = (uint64)val;
							if (hexDigits > 16)
								mPassInstance->FailAt("Too many hex digits for int64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
							else if ((hadOverflow) || (wasNeg))
								mPassInstance->FailAt("Value doesn't fit into uint64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
							mSyntaxToken = BfSyntaxToken_Literal;
							return;
						}
						mTokenEnd = mSrcIdx;
						mLiteral.mTypeCode = BfTypeCode_UIntPtr;
						mLiteral.mUInt32 = (uint32)val;
						if ((hadOverflow) || (wasNeg) || ((uint64)val != (uint64)mLiteral.mUInt32))
							mPassInstance->FailAt("Value doesn't fit into uint32", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
						mSyntaxToken = BfSyntaxToken_Literal;
						return;
					}
					else if ((c == 'l') || (c == 'L'))
					{
						if (c == 'l')
							TokenFail("Uppercase 'L' required for int64");
						if (wasNeg)
							val = -val;
						if ((mSrc[mSrcIdx] == 'u') || (mSrc[mSrcIdx] == 'U'))
						{
							mSrcIdx++;
							mTokenEnd = mSrcIdx;
							mLiteral.mTypeCode = BfTypeCode_UInt64;
							mLiteral.mUInt64 = (uint64)val;
							if (hexDigits > 16)
								mPassInstance->FailAt("Too many hex digits for int64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
							else if ((hadOverflow) || (wasNeg))
								mPassInstance->FailAt("Value doesn't fit into uint64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
							mSyntaxToken = BfSyntaxToken_Literal;
							return;
						}
						mTokenEnd = mSrcIdx;
						mLiteral.mTypeCode = BfTypeCode_Int64;
						mLiteral.mInt64 = (int64)val;

						bool signMatched = true;
						if (val != 0)
							signMatched = (val < 0) == wasNeg;

						if (hexDigits > 16)
							mPassInstance->FailAt("Too many hex digits for int64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
						else if ((hadOverflow) || (!signMatched))
							mPassInstance->FailAt("Value doesn't fit into int64", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
						mSyntaxToken = BfSyntaxToken_Literal;
						return;
					}
					else if ((c == 'f') || (c == 'F'))
					{
						mTokenEnd = mSrcIdx;
						mLiteral.mTypeCode = BfTypeCode_Single;
						mLiteral.mSingle = (float)ParseLiteralDouble();//(float)val;
						mSyntaxToken = BfSyntaxToken_Literal;
						return;
					}
					else if ((c == 'd') || (c == 'D'))
					{
						mTokenEnd = mSrcIdx;
						mLiteral.mTypeCode = BfTypeCode_Double;
						mLiteral.mDouble = ParseLiteralDouble();//(double)val;
						mSyntaxToken = BfSyntaxToken_Literal;
						return;
					}
					else
					{
						mTokenEnd = mSrcIdx - 1;
						mSrcIdx--;
						if (wasNeg)
							val = -val;
						mLiteral.mInt64 = val;
						mLiteral.mTypeCode = BfTypeCode_IntUnknown;
						mSyntaxToken = BfSyntaxToken_Literal;
						TokenFail("Unexpected character while parsing number", 0);
						return;
					}

					//if ((val < 0) && (val != -0x8000000000000000))
					if ((uint64)prevVal > (uint64)val)
						hadOverflow = true;
				}
			}
			else
			{
				if ((mCompatMode) && (c == '\\'))
				{
					int checkIdx = mSrcIdx;
					bool isAtLineEnd = true;
					while (true)
					{
						char checkC = mSrc[checkIdx];
						if ((checkC == '\r') || (checkC == '\n'))
							break;
						if ((checkC != ' ') && (checkC != '\t'))
						{
							isAtLineEnd = false;
							break;
						}
						checkIdx++;
					}

					if (isAtLineEnd)
						continue;
				}

				if (!isVerbatim)
				{
					switch (GetTokenHash())
					{
					case TOKEN_HASH('t', 'r', 'u', 'e'):
						if (SrcPtrHasToken("true"))
						{
							mLiteral.mTypeCode = BfTypeCode_Boolean;
							mLiteral.mInt64 = 1;
							mSyntaxToken = BfSyntaxToken_Literal;
							return;
						}
						break;
					case TOKEN_HASH('f', 'a', 'l', 's'):
						if (SrcPtrHasToken("false"))
						{
							mLiteral.mTypeCode = BfTypeCode_Boolean;
							mLiteral.mInt64 = 0;
							mSyntaxToken = BfSyntaxToken_Literal;
							return;
						}
						break;

					case TOKEN_HASH('a', 'b', 's', 't'):
						if ((!mCompatMode) && (SrcPtrHasToken("abstract")))
							mToken = BfToken_Abstract;
						break;
					case TOKEN_HASH('a', 'l', 'i', 'g'):
						if (SrcPtrHasToken("alignof"))
							mToken = BfToken_AlignOf;
						break;
					case TOKEN_HASH('a', 'p', 'p', 'e'):
						if ((!mCompatMode) && (SrcPtrHasToken("append")))
							mToken = BfToken_Append;
						break;
					case TOKEN_HASH('a', 's', 0, 0):
						if ((!mCompatMode) && (SrcPtrHasToken("as")))
							mToken = BfToken_As;
						break;
					case TOKEN_HASH('a', 's', 'm', 0):
						if (SrcPtrHasToken("asm"))
							mToken = BfToken_Asm;
						break;
					case TOKEN_HASH('b', 'a', 's', 'e'):
						if (SrcPtrHasToken("base"))
							mToken = BfToken_Base;
						break;
					case TOKEN_HASH('b', 'o', 'x', 0):
						if ((!mCompatMode) && (SrcPtrHasToken("box")))
							mToken = BfToken_Box;
						break;
					case TOKEN_HASH('b', 'r', 'e', 'a'):
						if (SrcPtrHasToken("break"))
							mToken = BfToken_Break;
						break;
					case TOKEN_HASH('c', 'a', 's', 'e'):
						if (SrcPtrHasToken("case"))
							mToken = BfToken_Case;
						break;
					case TOKEN_HASH('c', 'a', 't', 'c'):
						if (SrcPtrHasToken("catch"))
							mToken = BfToken_Catch;
						break;
					case TOKEN_HASH('c', 'h', 'e', 'c'):
						if ((!mCompatMode) && (SrcPtrHasToken("checked")))
							mToken = BfToken_Checked;
						break;
					case TOKEN_HASH('c', 'l', 'a', 's'):
						if (SrcPtrHasToken("class"))
							mToken = BfToken_Class;
						break;
					case TOKEN_HASH('c', 'o', 'n', 'c'):
						if ((!mCompatMode) && (SrcPtrHasToken("concrete")))
							mToken = BfToken_Concrete;
						break;
					case TOKEN_HASH('c', 'o', 'n', 's'):
						if (SrcPtrHasToken("const"))
							mToken = BfToken_Const;
						break;
					case TOKEN_HASH('c', 'o', 'n', 't'):
						if (SrcPtrHasToken("continue"))
							mToken = BfToken_Continue;
						break;
					case TOKEN_HASH('d', 'e', 'c', 'l'):
						if (SrcPtrHasToken("decltype"))
							mToken = BfToken_Decltype;
						break;
					case TOKEN_HASH('d', 'e', 'f', 'a'):
						if (SrcPtrHasToken("default"))
							mToken = BfToken_Default;
						break;
					case TOKEN_HASH('d', 'e', 'f', 'e'):
						if ((!mCompatMode) && (SrcPtrHasToken("defer")))
							mToken = BfToken_Defer;
						break;
					case TOKEN_HASH('d', 'e', 'l', 'e'):
						if ((!mCompatMode) && (SrcPtrHasToken("delegate")))
							mToken = BfToken_Delegate;
						else if (SrcPtrHasToken("delete"))
							mToken = BfToken_Delete;
						break;
					case TOKEN_HASH('d', 'o', 0, 0):
						if (SrcPtrHasToken("do"))
							mToken = BfToken_Do;
						break;
						break;
					case TOKEN_HASH('e', 'l', 's', 'e'):
						if (SrcPtrHasToken("else"))
							mToken = BfToken_Else;
						break;
					case TOKEN_HASH('e', 'n', 'u', 'm'):
						if (SrcPtrHasToken("enum"))
							mToken = BfToken_Enum;
						break;
					case TOKEN_HASH('e', 'x', 'p', 'l'):
						if ((!mCompatMode) && (SrcPtrHasToken("explicit")))
							mToken = BfToken_Explicit;
						break;
					case TOKEN_HASH('e', 'x', 't', 'e'):
						if (SrcPtrHasToken("extern"))
							mToken = BfToken_Extern;
						else if ((!mCompatMode) && (SrcPtrHasToken("extension")))
							mToken = BfToken_Extension;
						break;
					case TOKEN_HASH('f', 'a', 'l', 'l'):
						if ((!mCompatMode) && (SrcPtrHasToken("fallthrough")))
							mToken = BfToken_Fallthrough;
						break;
					case TOKEN_HASH('f', 'i', 'n', 'a'):
						if (SrcPtrHasToken("finally"))
							mToken = BfToken_Finally;
						break;
					case TOKEN_HASH('f', 'i', 'x', 'e'):
						if (SrcPtrHasToken("fixed"))
							mToken = BfToken_Fixed;
						break;
					case TOKEN_HASH('f', 'o', 'r', 0):
						if (SrcPtrHasToken("for"))
							mToken = BfToken_For;
						break;
					case TOKEN_HASH('f', 'o', 'r', 'e'):
						if ((!mCompatMode) && (SrcPtrHasToken("foreach")))
						{
							mToken = BfToken_For;
							mPassInstance->WarnAt(0, "'foreach' should be renamed to 'for'", mSourceData, mTokenStart, mSrcIdx - mTokenStart);
						}
						break;
					case TOKEN_HASH('f', 'u', 'n', 'c'):
						if ((!mCompatMode) && (SrcPtrHasToken("function")))
							mToken = BfToken_Function;
						break;
					case TOKEN_HASH('g', 'o', 't', 'o'):
						if (SrcPtrHasToken("goto"))
							mToken = BfToken_Goto;
						break;
					case TOKEN_HASH('i', 'f', 0, 0):
						if (SrcPtrHasToken("if"))
							mToken = BfToken_If;
						break;
					case TOKEN_HASH('i', 'm', 'p', 'l'):
						if ((!mCompatMode) && (SrcPtrHasToken("implicit")))
							mToken = BfToken_Implicit;
						break;
					case TOKEN_HASH('i', 'n', 0, 0):
						if ((!mCompatMode) && (SrcPtrHasToken("in")))
							mToken = BfToken_In;
						break;
					case TOKEN_HASH('i', 'n', 'l', 'i'):
						if ((!mCompatMode) && (SrcPtrHasToken("inline")))
							mToken = BfToken_Inline;
						break;
					case TOKEN_HASH('i', 'n', 't', 'e'):
						if ((!mCompatMode) && (SrcPtrHasToken("interface")))
							mToken = BfToken_Interface;
						else if ((!mCompatMode) && (SrcPtrHasToken("internal")))
							mToken = BfToken_Internal;
						break;
					case TOKEN_HASH('i', 's', 0, 0):
						if ((!mCompatMode) && (SrcPtrHasToken("is")))
							mToken = BfToken_Is;
						break;
					case TOKEN_HASH('l', 'e', 't', 0):
						if ((!mCompatMode) && (SrcPtrHasToken("let")))
							mToken = BfToken_Let;
						break;
					case TOKEN_HASH('m', 'i', 'x', 'i'):
						if ((!mCompatMode) && (SrcPtrHasToken("mixin")))
							mToken = BfToken_Mixin;
						break;
					case TOKEN_HASH('m', 'u', 't', 0):
						if ((!mCompatMode) && (SrcPtrHasToken("mut")))
							mToken = BfToken_Mut;
						break;
					case TOKEN_HASH('n', 'a', 'm', 'e'):
						if (SrcPtrHasToken("namespace"))
							mToken = BfToken_Namespace;
						break;
					case TOKEN_HASH('n', 'e', 'w', 0):
						if (SrcPtrHasToken("new"))
							mToken = BfToken_New;
						break;
					case TOKEN_HASH('n', 'u', 'l', 'l'):
						if (SrcPtrHasToken("null"))
							mToken = BfToken_Null;
						break;
					case TOKEN_HASH('o', 'p', 'e', 'r'):
						if (SrcPtrHasToken("operator"))
							mToken = BfToken_Operator;
						break;
					case TOKEN_HASH('o', 'u', 't', 0):
						if ((!mCompatMode) && (SrcPtrHasToken("out")))
							mToken = BfToken_Out;
						break;
					case TOKEN_HASH('o', 'v', 'e', 'r'):
						if (SrcPtrHasToken("override"))
							mToken = BfToken_Override;
						break;
					case TOKEN_HASH('p', 'a', 'r', 'a'):
						if ((!mCompatMode) && (SrcPtrHasToken("params")))
							mToken = BfToken_Params;
						break;					
					case TOKEN_HASH('p', 'r', 'i', 'v'):
						if (SrcPtrHasToken("private"))
							mToken = BfToken_Private;
						break;
					case TOKEN_HASH('p', 'r', 'o', 't'):
						if (SrcPtrHasToken("protected"))
							mToken = BfToken_Protected;
						break;
					case TOKEN_HASH('p', 'u', 'b', 'l'):
						if (SrcPtrHasToken("public"))
							mToken = BfToken_Public;
						break;
					case TOKEN_HASH('r', 'e', 'a', 'd'):
						if ((!mCompatMode) && (SrcPtrHasToken("readonly")))
							mToken = BfToken_ReadOnly;
						break;
					case TOKEN_HASH('r', 'e', 'f', 0):
						if ((!mCompatMode) && (SrcPtrHasToken("ref")))
							mToken = BfToken_Ref;
						break;
					case TOKEN_HASH('r', 'e', 'p', 'e'):
						if ((!mCompatMode) && (SrcPtrHasToken("repeat")))
							mToken = BfToken_Repeat;
						break;
					case TOKEN_HASH('r', 'e', 't', 't'):
						if ((!mCompatMode) && (SrcPtrHasToken("rettype")))
							mToken = BfToken_RetType;
						break;
					case TOKEN_HASH('r', 'e', 't', 'u'):
						if (SrcPtrHasToken("return"))
							mToken = BfToken_Return;
						break;
					case TOKEN_HASH('s', 'c', 'o', 'p'):
						if ((!mCompatMode) && (SrcPtrHasToken("scope")))
							mToken = BfToken_Scope;
						break;
					case TOKEN_HASH('s', 'e', 'a', 'l'):
						if ((!mCompatMode) && (SrcPtrHasToken("sealed")))
							mToken = BfToken_Sealed;
						break;
					case TOKEN_HASH('s', 'i', 'z', 'e'):
						if (SrcPtrHasToken("sizeof"))
							mToken = BfToken_SizeOf;
						break;
					case TOKEN_HASH('s', 't', 'a', 'c'):
						if ((!mCompatMode) && (SrcPtrHasToken("stack")))
							mToken = BfToken_Stack;
						break;
					case TOKEN_HASH('s', 't', 'a', 't'):
						if (SrcPtrHasToken("static"))
							mToken = BfToken_Static;
						break;
					case TOKEN_HASH('s', 't', 'r', 'i'):
						if (SrcPtrHasToken("strideof"))
							mToken = BfToken_StrideOf;
						break;
					case TOKEN_HASH('s', 't', 'r', 'u'):
						if (SrcPtrHasToken("struct"))
							mToken = BfToken_Struct;
						break;
					case TOKEN_HASH('s', 'w', 'i', 't'):
						if (SrcPtrHasToken("switch"))
							mToken = BfToken_Switch;
						break;
					case TOKEN_HASH('t', 'h', 'i', 's'):
						if (SrcPtrHasToken("this"))
							mToken = BfToken_This;
						break;
					case TOKEN_HASH('t', 'h', 'r', 'o'):
						if (SrcPtrHasToken("throw"))
							mToken = BfToken_Throw;
						break;
					case TOKEN_HASH('t', 'r', 'y', 0):
						if (SrcPtrHasToken("try"))
							mToken = BfToken_Try;
						break;
					case TOKEN_HASH('t', 'y', 'p', 'e'):
						if (SrcPtrHasToken("typeof"))
							mToken = BfToken_TypeOf;
						else if (SrcPtrHasToken("typealias"))
							mToken = BfToken_TypeAlias;
						break;
					case TOKEN_HASH('u', 'n', 'c', 'h'):
						if (SrcPtrHasToken("unchecked"))
							mToken = BfToken_Unchecked;
						break;
					case TOKEN_HASH('u', 'n', 's', 'i'):
						if (mCompatMode)
						{
							if (SrcPtrHasToken("unsigned"))
								mToken = BfToken_Unsigned;
						}
						break;
					case TOKEN_HASH('u', 's', 'i', 'n'):
						if (SrcPtrHasToken("using"))
							mToken = BfToken_Using;
						break;
					case TOKEN_HASH('v', 'a', 'r', 0):
						if ((!mCompatMode) && (SrcPtrHasToken("var")))
							mToken = BfToken_Var;
						break;
					case TOKEN_HASH('v', 'i', 'r', 't'):
						if (SrcPtrHasToken("virtual"))
							mToken = BfToken_Virtual;
						break;
					case TOKEN_HASH('v', 'o', 'l', 'a'):
						if (SrcPtrHasToken("volatile"))
							mToken = BfToken_Volatile;
						break;
					case TOKEN_HASH('w', 'h', 'e', 'n'):
						if (SrcPtrHasToken("when"))
							mToken = BfToken_When;
						break;
					case TOKEN_HASH('w', 'h', 'e', 'r'):
						if (SrcPtrHasToken("where"))
							mToken = BfToken_Where;
						break;
					case TOKEN_HASH('w', 'h', 'i', 'l'):
						if (SrcPtrHasToken("while"))
							mToken = BfToken_While;
						break;
					case TOKEN_HASH('y', 'i', 'e', 'l'):
						if ((!mCompatMode) && (SrcPtrHasToken("yield")))
							mToken = BfToken_Yield;
						break;
					}
				}
				if (mToken != BfToken_None)
				{
					mSyntaxToken = BfSyntaxToken_Token;
					return;
				}

				bool allowChar = false;
				if (mCompatMode)
					allowChar = (c == '$') || (c == '`');

				if ((uint8)c >= 0xC0)
				{
					int cLen = 0;
					mSrcIdx--;
					uint32 c32 = u8_toucs(mSrc + mSrcIdx, mOrigSrcLength - mSrcIdx, &cLen);
					mSrcIdx += cLen;

					utf8proc_category_t cat = utf8proc_category(c32);
					switch (cat)
					{
					case UTF8PROC_CATEGORY_LU:
					case UTF8PROC_CATEGORY_LL:
					case UTF8PROC_CATEGORY_LT:
					case UTF8PROC_CATEGORY_LM:
					case UTF8PROC_CATEGORY_LO:
					case UTF8PROC_CATEGORY_NL:
						allowChar = true;
					default: break;
					}
				}

				if ((allowChar) ||
					((c >= 'A') && (c <= 'Z')) ||
					((c >= 'a') && (c <= 'z')) ||
					(c == '_'))
				{
					if (isVerbatim)
						mTokenStart = verbatimStart;

					while (true)
					{
						int curSrcIdx = mSrcIdx;
						char c = mSrc[mSrcIdx++];
						bool isValidChar =
							(((c >= 'A') && (c <= 'Z')) || ((c >= 'a') && (c <= 'z')) || (c == '_') || ((c >= '0') && (c <= '9')));
						if (mCompatMode)
							isValidChar |= (c == '$') || (c == '`') || (c == '\'');

						if ((uint8)c >= 0xC0)
						{
							int cLen = 0;
							mSrcIdx--;
							uint32 c32 = u8_toucs(mSrc + mSrcIdx, mOrigSrcLength - mSrcIdx, &cLen);
							mSrcIdx += cLen;

							utf8proc_category_t cat = utf8proc_category(c32);
							switch (cat)
							{
							case UTF8PROC_CATEGORY_LU:
							case UTF8PROC_CATEGORY_LL:
							case UTF8PROC_CATEGORY_LT:
							case UTF8PROC_CATEGORY_LM:
							case UTF8PROC_CATEGORY_LO:
							case UTF8PROC_CATEGORY_NL:

							case UTF8PROC_CATEGORY_MN:
							case UTF8PROC_CATEGORY_MC:
							case UTF8PROC_CATEGORY_ND:
							case UTF8PROC_CATEGORY_PC:
							case UTF8PROC_CATEGORY_CF:
								isValidChar = true;
							default: break;
							}
						}

						if (!isValidChar)
						{
							mTokenEnd = curSrcIdx;
							mSrcIdx = curSrcIdx;
							mSyntaxToken = BfSyntaxToken_Identifier;
							return;
						}
					}

					mSyntaxToken = BfSyntaxToken_Identifier;
					return;
				}
				else
				{
					AddErrorNode(mTokenStart, mSrcIdx);
					mTriviaStart = mSrcIdx;
					TokenFail("Unexpected character");
					continue;
				}
			}
			return;
		}

		if ((setVerbatim) && (!isVerbatim))
		{
			isVerbatim = true;
			verbatimStart = mTokenStart;
		}
	}
}

static int gParseBlockIdx = 0;
static int gParseMemberIdx = 0;

void BfParser::ParseBlock(BfBlock* astNode, int depth)
{
	gParseBlockIdx++;
	int startParseBlockIdx = gParseBlockIdx;

	bool isAsmBlock = false;

	SizedArray<BfAstNode*, 32> childArr;

	while (true)
	{
		if ((mSyntaxToken == BfSyntaxToken_Token) && (mToken == BfToken_Asm))
		{
			if (isAsmBlock || mInAsmBlock)
				mPassInstance->Fail("Already inside an 'asm' block", astNode);
			else
				isAsmBlock = true;
		}

		NextToken();
		if (mPreprocessorIgnoredSectionNode != NULL)
		{
			if (mSyntaxToken != BfSyntaxToken_EOF)
				continue;
			mPreprocessorIgnoredSectionNode->SetSrcEnd(mSrcIdx);
		}

		if (mScanOnly)
		{
			if (mSyntaxToken == BfSyntaxToken_EOF)
				break;
			continue;
		}

		gParseMemberIdx++;
		int memberIdx = gParseMemberIdx;
		
		auto childNode = CreateNode();
		if (childNode == NULL)
			break;
		if (mSyntaxToken == BfSyntaxToken_EOF)
		{
			if (astNode != 0)
				Fail("Unexpected end of file");
			break;
		}

		if (mToken == BfToken_LBrace)
		{
			BfBlock* newBlock;
			BfInlineAsmStatement* asmBlock = nullptr;
			BfBlock* genBlock = nullptr;
			/*if (isAsmBlock)
			{
			asmBlock = mAlloc->Alloc<BfInlineAsmStatement>();
			asmBlock->mOpenBrace = (BfTokenNode*)CreateNode();
			newBlock = asmBlock;
			mInAsmBlock = true;
			isAsmBlock = false;
			}
			else*/
			{
				genBlock = mAlloc->Alloc<BfBlock>();
				genBlock->mOpenBrace = (BfTokenNode*)CreateNode();
				newBlock = genBlock;
			}
			newBlock->Init(this);
			ParseBlock(newBlock, depth + 1);
			if (mToken == BfToken_RBrace)
			{
				if (genBlock)
					genBlock->mCloseBrace = (BfTokenNode*)CreateNode();
				else if (asmBlock)
					asmBlock->mCloseBrace = (BfTokenNode*)CreateNode();

				newBlock->SetSrcEnd(mSrcIdx);
			}
			else
			{
				if (mSyntaxToken == BfSyntaxToken_EOF)
					mPassInstance->FailAfterAt("Expected '}'", mSourceData, newBlock->GetSrcEnd() - 1);
			}
			mInAsmBlock = false;
			astNode->Add(newBlock);
			childArr.push_back(newBlock);			
		}
		else if (mToken == BfToken_RBrace)
		{
			if (depth == 0)
				Fail("Unexpected ending brace");			
			break;
		}
		else
		{
			astNode->Add(childNode);			
			childArr.push_back(childNode);

			if ((mSyntaxToken == BfSyntaxToken_Token) && (mToken == BfToken_RBrace))
				break;
		}
	}

	astNode->Init(childArr, mAlloc);
}

const char* BfNodeToString(BfAstNode* node)
{
	static char str[256] = { 0 };
	strncpy(str, node->GetSourceData()->mSrc + node->GetSrcStart(), node->GetSrcLength());
	return str;
}

BfAstNode* BfParser::CreateNode()
{
	switch (mSyntaxToken)
	{
		case BfSyntaxToken_Token:
		{
			auto bfTokenNode = mAlloc->Alloc<BfTokenNode>();
			bfTokenNode->Init(this);
			bfTokenNode->SetToken(mToken);
			return bfTokenNode;
		}
	case BfSyntaxToken_Identifier:
		{
			//auto bfIdentifierNode = new BfIdentifierNode();
			auto bfIdentifierNode = mAlloc->Alloc<BfIdentifierNode>();
			bfIdentifierNode->Init(this);
			return bfIdentifierNode;
		}
	case BfSyntaxToken_Literal:
		{
			auto bfLiteralExpression = mAlloc->Alloc<BfLiteralExpression>();
			bfLiteralExpression->Init(this);
			bfLiteralExpression->mValue = mLiteral;
			mLiteral.mTypeCode = BfTypeCode_None;
			mLiteral.mWarnType = 0;
			return bfLiteralExpression;
		}
	default: break;
	}

	return NULL;
}

void BfParser::Parse(BfPassInstance* passInstance)
{
	BP_ZONE_F("BfParser::Parse %s", mFileName.c_str());

	mPassInstance = passInstance;

	if (mUsingCache)
	{
		mRootNode = mParserData->mRootNode;
		mSidechannelRootNode = mParserData->mSidechannelRootNode;
		mErrorRootNode = mParserData->mErrorRootNode;
		return;
	}

	mRootNode = mAlloc->Alloc<BfRootNode>();
	mRootNode->Init(this);
	mParserData->mRootNode = mRootNode;

	mSidechannelRootNode = mAlloc->Alloc<BfRootNode>();
	mSidechannelRootNode->Init(this);
	mParserData->mSidechannelRootNode = mSidechannelRootNode;

	mErrorRootNode = mAlloc->Alloc<BfRootNode>();
	mErrorRootNode->Init(this);
	mParserData->mErrorRootNode = mErrorRootNode;

	ParseBlock(mRootNode, 0);

	if (mPreprocessorNodeStack.size() > 0)
	{
		mPassInstance->Warn(0, "No matching #endif found", mPreprocessorNodeStack.back().first);
	}

	for (int i = 1; i < mJumpTableSize; i++)
		if (mJumpTable[i].mCharIdx == 0)
			mJumpTable[i] = mJumpTable[i - 1];

	if (mPassInstance->HasFailed())
		mParsingFailed = true;

	if ((mPassInstance->HasMessages()) || (mParsingFailed))
	{
		mParserData->mFailed = true; // Don't reuse cache if there were errors or warnings		
	}

	mPassInstance = NULL;
}

int BfParser::GetCharIdAtIndex(int findIndex)
{
	return mParserData->GetCharIdAtIndex(findIndex);
}

void BfParser::Close()
{
	BfSource::Close();

	BfLogSys(mSystem, "Parser %p closing. RefCount:%d Failed:%d\n", this, mParserData->mRefCount, mParserData->mFailed);

	if ((mParserData->mRefCount == 0) && (!mParserData->mFailed))
	{
		BF_ASSERT(mParserData->mDidReduce);

		AutoCrit autoCrit(gBfParserCache->mCritSect);
		BfParserCache::DataEntry dataEntry;
		dataEntry.mParserData = mParserData;
		if (gBfParserCache->mEntries.Add(dataEntry))
		{
			BfLogSys(mSystem, "Parser %p added to cache\n", this);
			mParserData->mRefCount++;
		}
		else
		{
			// It's possible two of the same entries were being parsed at the same time on different threads. 
			//  Just let the loser be deleted in the dtor
			BfLogSys(mSystem, "Duplicate parser %p not added to cache\n", this);
		}
	}	
}

void BfParser::GenerateAutoCompleteFrom(int srcPosition)
{
	BfSourcePositionFinder posFinder(this, srcPosition);
	posFinder.Visit(mRootNode);
	if (posFinder.mClosestElement != NULL)
	{
	}
}

void BfParser::ReportMemory(MemReporter* memReporter)
{
	//memReporter->Add("SmallAstAlloc", (int)mAlloc->mPages.size() * BfAstAllocManager::PAGE_SIZE);
	//memReporter->Add("LargeAstAlloc", mAlloc->mLargeAllocSizes);

// 	if (!mUsingCache)
// 		memReporter->AddBumpAlloc("AstAlloc", *mAlloc);
// 
// 	memReporter->Add("JumpTable", mJumpTableSize * sizeof(BfLineStartEntry));

	memReporter->Add(sizeof(BfParser));
	if (mParserData->mRefCount <= 0)
		mParserData->ReportMemory(memReporter);

// 	if (mSrcAllocSize > 0)
// 		memReporter->Add("Source", mSrcAllocSize);
}

class BfInnermostFinder : public BfElementVisitor
{
public:
	int mCursorIdx;
	BfAstNode* mFoundNode;

	BfInnermostFinder(int cursorIdx)
	{
		mFoundNode = NULL;
		mCursorIdx = cursorIdx;
	}

	virtual void Visit(BfAstNode* node) override
	{
		if ((node->Contains(mCursorIdx)) && (!node->IsA<BfBlock>()))
		{
			if ((mFoundNode == NULL) || ((node->GetSrcLength()) <= (mFoundNode->GetSrcLength())))
				mFoundNode = node;
		}
	}

	virtual void Visit(BfMemberReferenceExpression* memberRefExpr) override
	{
		BfElementVisitor::Visit(memberRefExpr);
		if (mFoundNode == memberRefExpr->mMemberName)
		{
			mFoundNode = memberRefExpr;
		}
	}

	virtual void Visit(BfAttributedIdentifierNode* identifierNode) override
	{
		BfElementVisitor::Visit(identifierNode);
		if (mFoundNode == identifierNode->mIdentifier)
		{
			mFoundNode = identifierNode;
		}
	}

	virtual void Visit(BfQualifiedNameNode* qualifiedNameNode) override
	{
		BfElementVisitor::Visit(qualifiedNameNode);
		if (mFoundNode == qualifiedNameNode->mRight)
		{
			mFoundNode = qualifiedNameNode;
		}
	}

	virtual void Visit(BfBinaryOperatorExpression* binaryOpExpr) override
	{
		BfElementVisitor::Visit(binaryOpExpr);
		if (mFoundNode == binaryOpExpr->mOpToken)
		{
			mFoundNode = binaryOpExpr;
		}
	}

	virtual void Visit(BfPreprocessorNode* preprocNode) override
	{
		if (preprocNode->mArgument != NULL)
		{
			for (auto arg : preprocNode->mArgument->mChildArr)
			{
				Visit((BfAstNode*)arg);
				if (mFoundNode == arg)
				{
					mFoundNode = preprocNode;
				}
			}
		}
	}
};

static BfAstNode* FindDebugExpressionNode(BfAstNode* checkNode, int cursorIdx)
{
	BfInnermostFinder innermostFinder(cursorIdx);
	innermostFinder.VisitChild(checkNode);
	BfAstNode* exprNode = innermostFinder.mFoundNode;
	return exprNode;
}

//////////////////////////////////////////////////////////////////////////

BF_EXPORT void BF_CALLTYPE BfParser_SetSource(BfParser* bfParser, const char* data, int length, const char* fileName)
{
	bfParser->mFileName = fileName;
	bfParser->SetSource(data, length);
}

BF_EXPORT void BF_CALLTYPE BfParser_SetCharIdData(BfParser* bfParser, uint8* data, int length)
{
	if (bfParser->mParserData->mCharIdData != NULL)
		delete bfParser->mParserData->mCharIdData;
	bfParser->mParserData->mCharIdData = new uint8[length];
	memcpy(bfParser->mParserData->mCharIdData, data, length);
}

BF_EXPORT void BF_CALLTYPE BfParser_SetHashMD5(BfParser* bfParser, Val128* md5Hash)
{
	if (md5Hash != NULL)
		bfParser->mParserData->mMD5Hash = *md5Hash;
}

BF_EXPORT void BF_CALLTYPE BfParser_Delete(BfParser* bfParser)
{
	if (bfParser->mNextRevision != NULL)
		bfParser->mNextRevision->mPrevRevision = NULL;
	auto itr = std::find(bfParser->mSystem->mParsers.begin(), bfParser->mSystem->mParsers.end(), bfParser);
	bfParser->mSystem->mParsers.erase(itr);
	delete bfParser;
}

BF_EXPORT void BF_CALLTYPE BfParser_SetNextRevision(BfParser* bfParser, BfParser* nextRevision)
{
	BF_ASSERT(bfParser->mNextRevision == NULL);
	BF_ASSERT(nextRevision->mPrevRevision == NULL);
	bfParser->mNextRevision = nextRevision;
	nextRevision->mPrevRevision = bfParser;

	nextRevision->mDataId = bfParser->mDataId;
}

BF_EXPORT void BF_CALLTYPE BfParser_SetCursorIdx(BfParser* bfParser, int cursorIdx)
{
	bfParser->SetCursorIdx(cursorIdx);	
}

BF_EXPORT void BF_CALLTYPE BfParser_SetIsClassifying(BfParser* bfParser)
{
	bfParser->mParserFlags = (BfParserFlag)(bfParser->mParserFlags | ParserFlag_Classifying);
}

BF_EXPORT void BF_CALLTYPE BfParser_SetAutocomplete(BfParser* bfParser, int cursorIdx)
{
	BF_ASSERT(bfParser->mParserData->mRefCount == -1);
	bfParser->SetCursorIdx(cursorIdx);
	bfParser->mParserFlags = (BfParserFlag)(bfParser->mParserFlags | ParserFlag_Autocomplete | ParserFlag_Classifying);
}

PerfManager* BfGetPerfManager(BfParser* bfParser)
{
	if (bfParser == NULL)
		return NULL;
	if (bfParser->mCursorIdx != -1)
		return gPerfManager;
	return NULL;
}

BF_EXPORT bool BF_CALLTYPE BfParser_Parse(BfParser* bfParser, BfPassInstance* bfPassInstance, bool compatMode)
{
	BP_ZONE("BfParser_Parse");
	int startFailIdx = bfPassInstance->mFailedIdx;
	bfParser->mCompatMode = compatMode;
	bfParser->mQuickCompatMode = compatMode;
	bfParser->Parse(bfPassInstance);
	return startFailIdx == bfPassInstance->mFailedIdx;
}

BF_EXPORT bool BF_CALLTYPE BfParser_Reduce(BfParser* bfParser, BfPassInstance* bfPassInstance)
{
	BP_ZONE("BfParser_Reduce");
	if (bfParser->mUsingCache)
		return true; // Already reduced
	
	bfParser->FinishSideNodes();
	int startFailIdx = bfPassInstance->mFailedIdx;
	int startWarningCount = bfPassInstance->mWarningCount;
	BfReducer bfReducer;
	bfReducer.mSource = bfParser;
	bfReducer.mCompatMode = bfParser->mCompatMode;
	bfReducer.mPassInstance = bfPassInstance;
	bfReducer.HandleRoot(bfParser->mRootNode);
	if ((startFailIdx != bfPassInstance->mFailedIdx) ||
		(startWarningCount != bfPassInstance->mWarningCount))
		bfParser->mParserData->mFailed = true;
	bfParser->mParserData->mDidReduce = true;
	bfParser->Close();

	return startFailIdx == bfPassInstance->mFailedIdx;
}

static Array<int> gCharMapping;
BF_EXPORT const char* BF_CALLTYPE BfParser_Format(BfParser* bfParser, int formatStart, int formatEnd, int** outCharMapping)
{
	BP_ZONE("BfParser_Reduce");
	String& outString = *gTLStrReturn.Get();
	outString.clear();
	gCharMapping.Clear();
	BfPrinter bfPrinter(bfParser->mRootNode, bfParser->mSidechannelRootNode, bfParser->mErrorRootNode);
	bfPrinter.mFormatStart = formatStart;
	bfPrinter.mFormatEnd = formatEnd;
	bfPrinter.mCharMapping = &gCharMapping;
	bfPrinter.Visit(bfParser->mRootNode);
	outString = bfPrinter.mOutString;
	*outCharMapping = &gCharMapping[0];
	return outString.c_str();
}

BF_EXPORT const char* BF_CALLTYPE BfParser_DocPrep(BfParser* bfParser)
{
	BP_ZONE("BfParser_Reduce");
	String& outString = *gTLStrReturn.Get();
	outString.clear();
	gCharMapping.Clear();
	BfPrinter bfPrinter(bfParser->mRootNode, bfParser->mSidechannelRootNode, bfParser->mErrorRootNode);
	bfPrinter.mFormatStart = -1;
	bfPrinter.mFormatEnd = -1;
	bfPrinter.mCharMapping = &gCharMapping;
	bfPrinter.mDocPrep = true;
	bfPrinter.Visit(bfParser->mRootNode);
	outString = bfPrinter.mOutString;	
	return outString.c_str();
}

BF_EXPORT const char* BF_CALLTYPE BfParser_GetDebugExpressionAt(BfParser* bfParser, int cursorIdx)
{
	BP_ZONE("BfParser_Reduce");
	String& outString = *gTLStrReturn.Get();
	outString.clear();

	BfAstNode* exprNode = FindDebugExpressionNode(bfParser->mRootNode, cursorIdx);
	if (exprNode == NULL)
		exprNode = FindDebugExpressionNode(bfParser->mSidechannelRootNode, cursorIdx);
	if (exprNode == NULL)
		return NULL;

	if ((exprNode->IsA<BfMethodDeclaration>()) ||
		(exprNode->IsA<BfBlock>()) ||
		(exprNode->IsA<BfStatement>()) ||
		(exprNode->IsA<BfTokenNode>())
		)
	{
		return NULL;
	}

	BfPrinter bfPrinter(bfParser->mRootNode, NULL, NULL);
	bfPrinter.mReformatting = true;
	bfPrinter.mIgnoreTrivia = true;
	bfPrinter.VisitChild(exprNode);
	outString = bfPrinter.mOutString;

	if (auto preprocessorNode = BfNodeDynCast<BfPreprocessorNode>(exprNode))
	{
		auto firstStr = preprocessorNode->mArgument->mChildArr[0]->ToString();

		if (firstStr == "warning")
		{
			String warningNumStr = preprocessorNode->mArgument->mChildArr.back()->ToString();
			int warningNum = atoi(warningNumStr.c_str());
			String warningStr;
			switch (warningNum)
			{
			case BfWarning_CS0108_MemberHidesInherited:
				warningStr = "CS0108: Derived member hides inherited member";
				break;
			case BfWarning_CS0114_MethodHidesInherited:
				warningStr = "CS0114: Derived method hides inherited member";
				break;
			case BfWarning_CS0162_UnreachableCode:
				warningStr = "CS0162: Unreachable code";
				break;
			case BfWarning_CS0168_VariableDeclaredButNeverUsed:
				warningStr = "CS0168: Variable declared but never used";
				break;
			case BfWarning_CS0472_ValueTypeNullCompare:
				warningStr = "CS0472: ValueType compared to null";
				break;
			case BfWarning_CS1030_PragmaWarning:
				warningStr = "CS1030: Pragma warning";
				break;
			}
			if (!warningStr.empty())
				outString = "`" + warningStr;
		}
	}

	return outString.c_str();
}

BF_EXPORT BfResolvePassData* BF_CALLTYPE BfParser_CreateResolvePassData(BfParser* bfParser, BfResolveType resolveType)
{
	auto bfResolvePassData = new BfResolvePassData();
	bfResolvePassData->mResolveType = resolveType;
	bfResolvePassData->mParser = bfParser;
	if ((bfParser != NULL) && ((bfParser->mParserFlags & ParserFlag_Autocomplete) != 0))
		bfResolvePassData->mAutoComplete = new BfAutoComplete(resolveType);
	return bfResolvePassData;
}

BF_EXPORT bool BF_CALLTYPE BfParser_BuildDefs(BfParser* bfParser, BfPassInstance* bfPassInstance, BfResolvePassData* resolvePassData, bool fullRefresh)
{	
	BP_ZONE("BfParser_BuildDefs");
	int startFailIdx = bfPassInstance->mFailedIdx;
	BfDefBuilder defBuilder(bfParser->mSystem);
	defBuilder.mResolvePassData = resolvePassData;
	defBuilder.Process(bfPassInstance, bfParser, fullRefresh);
	return startFailIdx == bfPassInstance->mFailedIdx;;
}

BF_EXPORT void BF_CALLTYPE BfParser_RemoveDefs(BfParser* bfParser)
{
	
}

BF_EXPORT void BF_CALLTYPE BfParser_ClassifySource(BfParser* bfParser, BfSourceClassifier::CharData* charData, bool preserveFlags)
{
	if (!bfParser->mIsClosed)
		bfParser->Close();

	BfSourceClassifier bfSourceClassifier(bfParser, charData);
	bfSourceClassifier.mPreserveFlags = preserveFlags;	
	bfSourceClassifier.Visit(bfParser->mRootNode);
	bfSourceClassifier.mIsSideChannel = false; //? false or true?
	bfSourceClassifier.Visit(bfParser->mErrorRootNode);
	bfSourceClassifier.mIsSideChannel = true;
	bfSourceClassifier.Visit(bfParser->mSidechannelRootNode);	
}

BF_EXPORT void BF_CALLTYPE BfParser_GenerateAutoCompletionFrom(BfParser* bfParser, int srcPosition)
{
	BP_ZONE("BfParser_GenerateAutoCompletionFrom");
	bfParser->GenerateAutoCompleteFrom(srcPosition);
}

BF_EXPORT void BF_CALLTYPE BfParser_SetCompleteParse(BfParser* bfParser)
{
	bfParser->mCompleteParse = true;
}