Use already injected RenderDoc dll, or load it from default location (#2561)

* Use already injected RenderDoc dll, or load it from default location

- on Windows, if the process was launched from RenderDoc and the dll is already injected, use it
- otherwise try to load the dll from the default installation path in Program Files
- doesn't need the dll to be copied next to the exe or in the system PATH, which is not the supported way to do it according to BaldurK - see https://github.com/baldurk/renderdoc/issues/2279#issuecomment-844588691

* Restore previous implementation of findModule

* Address PR feedback

Co-authored-by: Nathan Reed <[email protected]>

src/debug_renderdoc.cpp

@@ -13,9 +13,11 @@
 
 namespace bgfx
 {
-	bool findModule(const char* _name)
+	void* findModule(const char* _name)
 	{
 #if BX_PLATFORM_WINDOWS
+		// NOTE: there was some reason to do it this way instead of simply calling GetModuleHandleA,
+		// but not sure what it was.
 		HANDLE process = GetCurrentProcess();
 		DWORD size;
 		BOOL result = EnumProcessModules(process
@@ -45,14 +47,16 @@ namespace bgfx
 					if (0 != result
 					&&  0 == bx::strCmpI(_name, moduleName) )
 					{
-						return true;
+						return (void*)modules[ii];
 					}
 				}
 			}
 		}
-#endif // BX_PLATFORM_WINDOWS
+#else
 		BX_UNUSED(_name);
-		return false;
+#endif // BX_PLATFORM_WINDOWS
+
+		return NULL;
 	}
 
 	pRENDERDOC_GetAPI RENDERDOC_GetAPI;
@@ -72,13 +76,19 @@ namespace bgfx
 			return NULL;
 		}
 
-		void* renderDocDll = bx::dlopen(
+		// If RenderDoc is already injected in the process then use the already present DLL
+		void* renderDocDll = findModule("renderdoc.dll");
+		if (NULL == renderDocDll)
+		{
+			// TODO: try common installation paths before looking in current directory
+			renderDocDll = bx::dlopen(
 #if BX_PLATFORM_WINDOWS
-				"renderdoc.dll"
+					"renderdoc.dll"
 #else
-				"./librenderdoc.so"
+					"./librenderdoc.so"
 #endif // BX_PLATFORM_WINDOWS
-				);
+					);
+		}
 
 		if (NULL != renderDocDll)
 		{
@@ -89,7 +99,7 @@ namespace bgfx
 			{
 				s_renderDoc->SetCaptureFilePathTemplate(BGFX_CONFIG_RENDERDOC_LOG_FILEPATH);
 
- 				s_renderDoc->SetFocusToggleKeys(NULL, 0);
+				s_renderDoc->SetFocusToggleKeys(NULL, 0);
 
 				RENDERDOC_InputButton captureKeys[] = BGFX_CONFIG_RENDERDOC_CAPTURE_KEYS;
 				s_renderDoc->SetCaptureKeys(captureKeys, BX_COUNTOF(captureKeys) );