Moved hooks calls and added hook retry

With that change, I no longer re-hook rendering functions as Hook_Manager::FoundRenderer removes all hooks and then we hook the true functions in the renderer specialization.
Added a retry count.

overlay_experimental/DX10_Hook.cpp

@@ -14,7 +14,8 @@ bool DX10_Hook::start_hook()
 {
     if (!_hooked)
     {
-        Hook_Manager::Inst().FoundRenderer(this);
+        if (!Windows_Hook::Inst().start_hook())
+            return false;
 
         IDXGISwapChain* pSwapChain;
         ID3D10Device* pDevice;
@@ -37,8 +38,11 @@ bool DX10_Hook::start_hook()
 
         if (pDevice != nullptr && pSwapChain != nullptr)
         {
-            _hooked = true;
             PRINT_DEBUG("Hooked DirectX 10\n");
+
+            _hooked = true;
+            Hook_Manager::Inst().FoundRenderer(this);
+            
             loadFunctions(pDevice, pSwapChain);
 
             UnhookAll();
@@ -50,7 +54,6 @@ bool DX10_Hook::start_hook()
             );
             EndHook();
 
-            if (Windows_Hook::Inst().start_hook())
             get_steam_client()->steam_overlay->HookReady();
         }
         else

overlay_experimental/DX11_Hook.cpp

@@ -24,7 +24,8 @@ bool DX11_Hook::start_hook()
 {
     if (!_hooked)
     {
-        Hook_Manager::Inst().FoundRenderer(this);
+        if (!Windows_Hook::Inst().start_hook())
+            return false;
 
         IDXGISwapChain* pSwapChain;
         ID3D11Device* pDevice;
@@ -47,8 +48,11 @@ bool DX11_Hook::start_hook()
 
         if (pDevice != nullptr && pSwapChain != nullptr)
         {
-            _hooked = true;
             PRINT_DEBUG("Hooked DirectX 11\n");
+
+            _hooked = true;
+            Hook_Manager::Inst().FoundRenderer(this);
+
             loadFunctions(pDevice, pSwapChain);
 
             UnhookAll();
@@ -60,7 +64,6 @@ bool DX11_Hook::start_hook()
             );
             EndHook();
 
-            if (Windows_Hook::Inst().start_hook())
             get_steam_client()->steam_overlay->HookReady();
         }
         else

overlay_experimental/DX12_Hook.cpp

@@ -14,6 +14,9 @@ bool DX12_Hook::start_hook()
 {
     if (!_hooked)
     {
+        //if (!Windows_Hook::Inst().start_hook())
+        //    return false;
+
         PRINT_DEBUG("Hooked DirectX 12\n");
         return false;
     }

overlay_experimental/DX9_Hook.cpp

@@ -16,7 +16,8 @@ bool DX9_Hook::start_hook()
 {
     if (!_hooked)
     {
-        Hook_Manager::Inst().FoundRenderer(this);
+        if (!Windows_Hook::Inst().start_hook())
+            return false;
 
         IDirect3D9Ex* pD3D;
         IDirect3DDevice9Ex* pDeviceEx;
@@ -33,8 +34,11 @@ bool DX9_Hook::start_hook()
 
         if (pDeviceEx != nullptr)
         {
-            _hooked = true;
             PRINT_DEBUG("Hooked DirectX 9\n");
+
+            _hooked = true;
+            Hook_Manager::Inst().FoundRenderer(this);
+            
             loadFunctions(pDeviceEx);
 
             UnhookAll();
@@ -47,7 +51,6 @@ bool DX9_Hook::start_hook()
             );
             EndHook();
 
-            if (Windows_Hook::Inst().start_hook())
             get_steam_client()->steam_overlay->HookReady();
         }
         else

overlay_experimental/Hook_Manager.cpp

@@ -29,111 +29,73 @@ decltype(wglMakeCurrent)* _wglMakeCurrent;
 
 HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain* _this, UINT SyncInterval, UINT Flags)
 {
-    IUnknown *pDevice;
+    Hook_Manager& inst = Hook_Manager::Inst();
-    _this->GetDevice(__uuidof(ID3D10Device), (void**)&pDevice);
+    if (!inst.stop_retry())
+    {
+        IUnknown* pDevice;
+        _this->GetDevice(__uuidof(ID3D10Device), (void**)& pDevice);
         if (pDevice)
         {
-        Hook_Manager::Inst().UnHookAllRendererDetector();
             DX10_Hook* hook = DX10_Hook::Inst();
-        if (!hook->start_hook())
+            if (hook->start_hook())
-        {
+                inst.AddHook(hook);
-            // Hook failed, start over
-            delete static_cast<Base_Hook*>(hook);
-            Hook_Manager::Inst().HookRenderer();
-        }
-        else
-        {
-            Hook_Manager::Inst().AddHook(hook);
-        }
         }
         else
         {
             _this->GetDevice(__uuidof(ID3D11Device), (void**)& pDevice);
             if (pDevice)
             {
-            Hook_Manager::Inst().UnHookAllRendererDetector();
                 DX11_Hook* hook = DX11_Hook::Inst();
-            if (!hook->start_hook())
+                if (hook->start_hook())
-            {
+                    inst.AddHook(hook);
-                // Hook failed, start over
-                delete static_cast<Base_Hook*>(hook);
-                Hook_Manager::Inst().HookRenderer();
-            }
-            else
-            {
-                Hook_Manager::Inst().AddHook(hook);
-            }
             }
             else
             {
                 _this->GetDevice(__uuidof(ID3D12Device), (void**)& pDevice);
                 DX12_Hook* hook = DX12_Hook::Inst();
-            if (!hook->start_hook())
+                if (hook->start_hook())
-            {
+                    inst.AddHook(hook);
-                // Hook failed, start over
-                delete static_cast<Base_Hook*>(hook);
-                Hook_Manager::Inst().HookRenderer();
-            }
-            else
-            {
-                Hook_Manager::Inst().AddHook(hook);
-            }
             }
         }
         if (pDevice) pDevice->Release();
+    }
 
     return (_this->*_IDXGISwapChain_Present)(SyncInterval, Flags);
 }
 
 HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion)
 {
-    Hook_Manager::Inst().UnHookAllRendererDetector();
+    Hook_Manager& inst = Hook_Manager::Inst();
+    if (!inst.stop_retry())
+    {
         DX9_Hook* hook = DX9_Hook::Inst();
-    if (!hook->start_hook())
+        if (hook->start_hook())
-    {
+            inst.AddHook(hook);
-        // Hook failed, start over
-        delete static_cast<Base_Hook*>(hook);
-        Hook_Manager::Inst().HookRenderer();
-    }
-    else
-    {
-        Hook_Manager::Inst().AddHook(hook);
     }
     return (_this->*_IDirect3DDevice9_Present)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion);
 }
 
 HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags)
 {
-    Hook_Manager::Inst().UnHookAllRendererDetector();
+    Hook_Manager& inst = Hook_Manager::Inst();
+    if (!inst.stop_retry())
+    {
         DX9_Hook* hook = DX9_Hook::Inst();
-    if (!hook->start_hook())
+        if (hook->start_hook())
-    {
+            inst.AddHook(hook);
-        // Hook failed, start over
-        delete static_cast<Base_Hook*>(hook);
-        Hook_Manager::Inst().HookRenderer();
-    }
-    else
-    {
-        Hook_Manager::Inst().AddHook(hook);
     }
     return (_this->*_IDirect3DDevice9Ex_PresentEx)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags);
 }
 
 BOOL WINAPI Hook_Manager::MywglMakeCurrent(HDC hDC, HGLRC hGLRC)
 {
-    Hook_Manager::Inst().UnHookAllRendererDetector();
+    Hook_Manager& inst = Hook_Manager::Inst();
+    if (!inst.stop_retry())
+    {
         OpenGL_Hook* hook = OpenGL_Hook::Inst();
-    if (!hook->start_hook())
+        if (hook->start_hook())
-    {
+            inst.AddHook(hook);
-        // Hook failed, start over
-        delete static_cast<Base_Hook*>(hook);
-        Hook_Manager::Inst().HookRenderer();
     }
-    else
-    {
-        Hook_Manager::Inst().AddHook(hook);
-    }
-
     return _wglMakeCurrent(hDC, hGLRC);
 }
 
@@ -369,6 +331,20 @@ HMODULE WINAPI Hook_Manager::MyLoadLibraryExW(LPCWSTR lpLibFileName, HANDLE hFil
     return res;
 }
 
+bool Hook_Manager::stop_retry()
+{
+    // Retry 200 times, we look for rendering functions so its actually: "retry for 200 frames"
+    bool stop = ++_hook_retries >= 200;
+
+    if (stop)
+    {
+        PRINT_DEBUG("We found a renderer but couldn't hook it, aborting overlay hook.\n");
+        FoundRenderer(nullptr);
+    }
+
+    return stop;
+}
+
 void Hook_Manager::HookLoadLibrary()
 {
     if (!_renderer_found && !_loadlibrary_hooked)
@@ -393,24 +369,8 @@ void Hook_Manager::HookLoadLibrary()
 
 #endif
 
-void Hook_Manager::UnHookAllRendererDetector()
-{
-    auto it = std::find(_hooks.begin(), _hooks.end(), rendererdetect_hook);
-    if (it != _hooks.end())
-    {
-        _hooks.erase(it);
-        delete rendererdetect_hook;
-        rendererdetect_hook = nullptr;
-    }
-    _loadlibrary_hooked = false;
-    _ogl_hooked = false;
-
-#ifdef STEAM_WIN32
-    _dxgi_hooked = _dx9_hooked = false;
-#endif
-}
-
 Hook_Manager::Hook_Manager():
+    _hook_retries(0),
 #ifdef STEAM_WIN32
     _loadlibrary_hooked(false),
     _dxgi_hooked(false),

overlay_experimental/Hook_Manager.h

@@ -31,6 +31,7 @@ protected:
     // If you do that, you should consider moving the renderer hooks to its own class and keep this one generic ?
     std::vector<Base_Hook*> _hooks; 
 
+    unsigned int _hook_retries;
     bool _renderer_found;       // Is the renderer hooked ?
     bool _ogl_hooked;           // wglMakeCurrent is hooked ? (opengl)
     Base_Hook* rendererdetect_hook;
@@ -39,10 +40,10 @@ protected:
     Hook_Manager();
     virtual ~Hook_Manager();
     
-    void UnHookAllRendererDetector();
     // Setup opengl device
     void hook_opengl();
 
+    bool stop_retry();
     void HookLoadLibrary();
 
 #if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64)

overlay_experimental/OpenGL_Hook.cpp

@@ -18,14 +18,20 @@ bool OpenGL_Hook::start_hook()
 {
     if (!_hooked)
     {
-        Hook_Manager::Inst().FoundRenderer(this);
+        if (!Windows_Hook::Inst().start_hook())
+            return false;
 
         GLenum err = glewInit();
 
         if (err == GLEW_OK)
         {
-            _hooked = true;
             PRINT_DEBUG("Hooked OpenGL\n");
+
+            _hooked = true;
+            Hook_Manager::Inst().FoundRenderer(this);
+
+            wglSwapBuffers = (decltype(wglSwapBuffers))GetProcAddress(reinterpret_cast<HMODULE>(_library), "wglSwapBuffers");
+
             UnhookAll();
             BeginHook();
             HookFuncs(
@@ -33,7 +39,6 @@ bool OpenGL_Hook::start_hook()
             );
             EndHook();
 
-            if (Windows_Hook::Inst().start_hook())
             get_steam_client()->steam_overlay->HookReady();
         }
         else
@@ -62,7 +67,6 @@ void OpenGL_Hook::resetRenderState()
 // Try to make this function and overlay's proc as short as possible or it might affect game's fps.
 void OpenGL_Hook::prepareForOverlay(HDC hDC)
 {
-
     HWND hWnd = WindowFromDC(hDC);
     RECT rect;
 

overlay_experimental/Windows_Hook.h

@@ -11,7 +11,7 @@
 class Windows_Hook : public Base_Hook
 {
 public:
-    //static constexpr const char* DLL_NAME = "user32.dll";
+    static constexpr const char* DLL_NAME = "user32.dll";
 
 private:
     // Variables