Improved hot compilation handling of DLLs

IDE/src/BuildContext.bf

@@ -16,6 +16,7 @@ namespace IDE
 		case DebugAfter;
 		case DebugComptime;
 		case Test;
+		case WhileRunning;
 
 		public bool WantsRunAfter
 		{

IDE/src/Debugger/DebugManager.bf

@@ -25,7 +25,8 @@ namespace IDE.Debugger
 			Terminating,
 			Terminated,
 			SearchingSymSrv,
-			HotResolve
+			HotResolve,
+			TargetUnloaded
 		}
 
 		public enum IntDisplayType
@@ -172,6 +173,9 @@ namespace IDE.Debugger
 		[CallingConvention(.Stdcall),CLink]
 		static extern RunState Debugger_GetRunState();
 
+		[CallingConvention(.Stdcall),CLink]
+		static extern bool Debugger_HasLoadedTargetBinary();
+
 		[CallingConvention(.Stdcall),CLink]
 		static extern char8* Debugger_GetCurrentException();
 
@@ -534,6 +538,11 @@ namespace IDE.Debugger
 			return Debugger_GetRunState();
 		}
 
+		public bool HasLoadedTargetBinary()
+		{
+			return Debugger_HasLoadedTargetBinary();
+		}
+
 		public bool HasPendingDebugLoads()
 		{
 			return Debugger_HasPendingDebugLoads();
@@ -561,8 +570,8 @@ namespace IDE.Debugger
 		public bool IsPaused(bool allowDebugEvalDone = false)
 		{
 			RunState runState = GetRunState();
-			return (runState == RunState.Paused) || (runState == RunState.Breakpoint) || (runState == RunState.Exception) ||
-				((runState == RunState.DebugEval_Done) && (allowDebugEvalDone));
+			return (runState == .Paused) || (runState == .Breakpoint) || (runState == .Exception) || 
+				((runState == .DebugEval_Done) && (allowDebugEvalDone));
 		}
 
 		public void GetCurrentException(String exStr)

IDE/src/IDEApp.bf

@@ -4501,6 +4501,10 @@ namespace IDE
 			        Compile(compileKind, null);
 			    }
 			}
+			else if ((mDebugger.mIsRunning) && (!mDebugger.HasLoadedTargetBinary()))
+			{
+				Compile(.WhileRunning, null);
+			}
 			else
 			{
 			    mOutputPanel.Clear();
@@ -11588,7 +11592,7 @@ namespace IDE
             if (mExecutionQueue.Count > 0)
                 return false;
 
-            if ((mDebugger != null) && (mDebugger.mIsRunning) && (hotProject == null))
+            if ((mDebugger != null) && (mDebugger.mIsRunning) && (hotProject == null) && (compileKind != .WhileRunning))
 			{
 				Debug.Assert(!mDebugger.mIsRunningCompiled);
 				Debug.Assert((compileKind == .Normal) || (compileKind == .DebugComptime));
@@ -13416,8 +13420,9 @@ namespace IDE
 							mBreakpointPanel.MarkStatsDirty();
                             mTargetHadFirstBreak = true;
 
-                            if (mDebugger.GetRunState() == DebugManager.RunState.Exception)
+                            switch (mDebugger.GetRunState())
                             {
+							case .Exception:
                                 String exceptionLine = scope String();
                                 mDebugger.GetCurrentException(exceptionLine);
                                 var exceptionData = String.StackSplit!(exceptionLine, '\n');
@@ -13431,11 +13436,26 @@ namespace IDE
                                 OutputLine(exString);
 								if (!IsCrashDump)
                                 	Fail(exString);
+							default:
                             }
                         }
                     }
                     break;
                 }
+				else if ((mDebugger != null) && (mDebugger.GetRunState() == .TargetUnloaded))
+				{
+					if (mWorkspace.mHadHotCompileSinceLastFullCompile)
+					{
+						// Had hot compiles - we need to recompile!
+						Compile(.WhileRunning);
+					}
+					else
+					{
+						mDebugger.Continue();
+					}
+					mWorkspace.StoppedRunning();
+					break;
+				}
                 else if (mDebuggerPerformingTask)
 				{
 					break;

IDEHelper/DebugManager.cpp

@@ -1108,6 +1108,14 @@ BF_EXPORT int BF_CALLTYPE Debugger_GetRunState()
 	return gDebugger->mRunState;
 }
 
+BF_EXPORT bool Debugger_HasLoadedTargetBinary()
+{
+	AutoCrit autoCrit(gDebugManager->mCritSect);
+	if (gDebugger == NULL)
+		return false;
+	return gDebugger->HasLoadedTargetBinary();
+}
+
 BF_EXPORT bool BF_CALLTYPE Debugger_HasPendingDebugLoads()
 {
 	if (gDebugger == NULL)

IDEHelper/DebugTarget.cpp

@@ -30,6 +30,8 @@ DebugTarget::DebugTarget(WinDebugger* debugger)
 	mCapturedNamesPtr = NULL;
 	mCapturedTypesPtr = NULL;
 	mHotHeap = NULL;
+	mHotHeapAddr = 0;
+	mHotHeapReserveSize = 0;
 	mLastHotHeapCleanIdx = 0;
 	mIsEmpty = false;
 	mWasLocallyBuilt = false;
@@ -107,12 +109,18 @@ void DebugTarget::SetupTargetBinary()
 		{
 			reservedPtr = (addr_target)VirtualAllocEx(mDebugger->mProcessInfo.hProcess, (void*)(intptr)checkHotReserveAddr, reserveSize, MEM_RESERVE, PAGE_EXECUTE_READWRITE);
 			if (reservedPtr != NULL)
+			{
+				mHotHeapAddr = reservedPtr;
+				mHotHeapReserveSize = reserveSize;
+				BfLogDbg("VirtualAllocEx %p %d. ImageBase: %p\n", mHotHeapAddr, mHotHeapReserveSize, mTargetBinary->mImageBase);
 				break;
+			}
 			checkHotReserveAddr += 4 * mb;
 		}
 
 		if (reservedPtr == 0)
 		{
+			BfLogDbg("VirtualAllocEx failed. ImageBase: %p\n", mTargetBinary->mImageBase);
 			mDebugger->Fail("Failed to reserve memory for hot swapping");
 		}
 		else
@@ -207,7 +215,7 @@ DbgModule* DebugTarget::SetupDyn(const StringImpl& filePath, DataStream* stream,
 {
 	BP_ZONE("DebugTarget::SetupDyn");
 
-	AutoDbgTime dbgTime("DebugTarget::SetupDyn " + filePath);
+	//AutoDbgTime dbgTime("DebugTarget::SetupDyn " + filePath);
 
 	DbgModule* dwarf = new COFF(this);
 	dwarf->mFilePath = filePath;
@@ -239,7 +247,7 @@ String DebugTarget::UnloadDyn(addr_target imageBase)
 {
 	String filePath;
 
-	AutoDbgTime dbgTime("DebugTarget::UnloadDyn");
+	//AutoDbgTime dbgTime("DebugTarget::UnloadDyn");
 
 	for (int i = 0; i < (int)mDbgModules.size(); i++)
 	{
@@ -252,7 +260,18 @@ String DebugTarget::UnloadDyn(addr_target imageBase)
 			filePath = dwarf->mFilePath;
 
 			if (mTargetBinary == dwarf)
+			{
 				mTargetBinary = NULL;
+				delete mHotHeap;
+				mHotHeap = NULL;
+
+				if (mHotHeapAddr != 0)
+				{
+					BfLogDbg("VirtualFreeEx %p %d\n", mHotHeapAddr, mHotHeapReserveSize);
+					::VirtualFreeEx(mDebugger->mProcessInfo.hProcess, (void*)(intptr)mHotHeapAddr, 0, MEM_RELEASE);
+					mHotHeapAddr = 0;
+				}
+			}
 
 			mDbgModules.RemoveAt(i);
 			bool success = mDbgModuleMap.Remove(dwarf->mId);

IDEHelper/DebugTarget.h

@@ -34,6 +34,8 @@ public:
 	Array<DbgType*>* mCapturedTypesPtr;
 
 	HotHeap* mHotHeap;
+	addr_target mHotHeapAddr;
+	int64 mHotHeapReserveSize;
 	int mLastHotHeapCleanIdx;
 	String mTargetPath;
 	DbgModule* mLaunchBinary;

IDEHelper/Debugger.h

@@ -159,7 +159,8 @@ enum RunState
 	RunState_Terminating,
 	RunState_Terminated,
 	RunState_SearchingSymSrv,
-	RunState_HotResolve
+	RunState_HotResolve,
+	RunState_TargetUnloaded
 };
 
 enum DebuggerResult
@@ -268,6 +269,7 @@ public:
 	virtual void OpenFile(const StringImpl& launchPath, const StringImpl& targetPath, const StringImpl& args, const StringImpl& workingDir, const Array<uint8>& envBlock, bool hotSwapEnabled) = 0;
 	virtual bool Attach(int processId, BfDbgAttachFlags attachFlags) = 0;
 	virtual void Run() = 0;
+	virtual bool HasLoadedTargetBinary() { return true; }
 	virtual void HotLoad(const Array<String>& objectFiles, int hotIdx) = 0;
 	virtual void InitiateHotResolve(DbgHotResolveFlags flags) = 0;
 	virtual intptr GetDbgAllocHeapSize() = 0;

IDEHelper/WinDebugger.cpp

@@ -1067,6 +1067,13 @@ void WinDebugger::Run()
 	CloseHandle(hThread);
 }
 
+bool WinDebugger::HasLoadedTargetBinary()
+{
+	if (mDebugTarget == NULL)
+		return false;
+	return mDebugTarget->mTargetBinary != NULL;
+}
+
 void WinDebugger::HotLoad(const Array<String>& objectFiles, int hotIdx)
 {
 	AutoCrit autoCrit(mDebugManager->mCritSect);
@@ -1891,10 +1898,16 @@ bool WinDebugger::DoUpdate()
 					}
 				}
 
+				bool hadTarget = mDebugTarget->mTargetBinary != NULL;
 				mDebugTarget->UnloadDyn(dbgModule->mImageBase);
 				if (needsBreakpointRehup)
 					RehupBreakpoints(true);
 
+				if ((mDebugTarget->mTargetBinary == NULL) && (hadTarget))
+				{
+					mRunState = RunState_TargetUnloaded;
+				}
+
 				mPendingDebugInfoLoad.Remove(dbgModule);
 				mPendingDebugInfoRequests.Remove(dbgModule);
 				mDebugManager->mOutMessages.push_back("modulesChanged");
@@ -2880,7 +2893,7 @@ void WinDebugger::ContinueDebugEvent()
 		}
 	}
 
-	if ((mRunState == RunState_Breakpoint) || (mRunState == RunState_Paused))
+	if ((mRunState == RunState_Breakpoint) || (mRunState == RunState_Paused) || (mRunState == RunState_TargetUnloaded))
 	{
 		ClearCallStack();
 		mRunState = RunState_Running;

IDEHelper/WinDebugger.h

@@ -584,6 +584,7 @@ public:
 	virtual void OpenFile(const StringImpl& launchPath, const StringImpl& targetPath, const StringImpl& args, const StringImpl& workingDir, const Array<uint8>& envBlock, bool hotSwapEnabled) override;
 	virtual bool Attach(int processId, BfDbgAttachFlags attachFlags) override;
 	virtual void Run() override;
+	virtual bool HasLoadedTargetBinary() override;
 	virtual void HotLoad(const Array<String>& objectFiles, int hotIdx) override;
 	virtual void InitiateHotResolve(DbgHotResolveFlags flags) override;
 	virtual intptr GetDbgAllocHeapSize() override;