Added error handling when hook fails

This commit is contained in:
Nemirtingas 2019-08-16 19:10:12 +02:00
parent 71d3e301c2
commit d06fbba104
12 changed files with 122 additions and 43 deletions

View file

@ -12,7 +12,7 @@
// This is created by DX10_Hook::Create, and deleted by the Hook_Manager if not used // This is created by DX10_Hook::Create, and deleted by the Hook_Manager if not used
static DX10_Hook* hook; static DX10_Hook* hook;
void DX10_Hook::start_hook() bool DX10_Hook::start_hook()
{ {
if (!_hooked) if (!_hooked)
{ {
@ -55,10 +55,12 @@ void DX10_Hook::start_hook()
else else
{ {
PRINT_DEBUG("Failed to hook DirectX 10\n"); PRINT_DEBUG("Failed to hook DirectX 10\n");
return false;
} }
if(pDevice)pDevice->Release(); if(pDevice)pDevice->Release();
if(pSwapChain)pSwapChain->Release(); if(pSwapChain)pSwapChain->Release();
} }
return true;
} }
void DX10_Hook::resetRenderState() void DX10_Hook::resetRenderState()

View file

@ -22,7 +22,6 @@ private:
DX10_Hook(); DX10_Hook();
virtual ~DX10_Hook(); virtual ~DX10_Hook();
void start_hook();
void resetRenderState(); void resetRenderState();
void prepareForOverlay(IDXGISwapChain *pSwapChain); void prepareForOverlay(IDXGISwapChain *pSwapChain);
@ -43,7 +42,8 @@ private:
//decltype(D3D10CreateDeviceAndSwapChain)* D3D10CreateDeviceAndSwapChain; //decltype(D3D10CreateDeviceAndSwapChain)* D3D10CreateDeviceAndSwapChain;
public: public:
static void Create(); // Initialize DX10 Hook. bool start_hook();
static DX10_Hook* Inst();
void loadFunctions(ID3D10Device *pDevice, IDXGISwapChain *pSwapChain); void loadFunctions(ID3D10Device *pDevice, IDXGISwapChain *pSwapChain);
}; };

View file

@ -22,7 +22,7 @@ HRESULT GetDeviceAndCtxFromSwapchain(IDXGISwapChain* pSwapChain, ID3D11Device**
return ret; return ret;
} }
void DX11_Hook::start_hook() bool DX11_Hook::start_hook()
{ {
if (!_hooked) if (!_hooked)
{ {
@ -65,11 +65,13 @@ void DX11_Hook::start_hook()
else else
{ {
PRINT_DEBUG("Failed to hook DirectX 11\n"); PRINT_DEBUG("Failed to hook DirectX 11\n");
return false;
} }
if(pDevice) pDevice->Release(); if(pDevice) pDevice->Release();
if(pSwapChain) pSwapChain->Release(); if(pSwapChain) pSwapChain->Release();
} }
return true;
} }
void DX11_Hook::resetRenderState() void DX11_Hook::resetRenderState()

View file

@ -22,7 +22,6 @@ private:
DX11_Hook(); DX11_Hook();
virtual ~DX11_Hook(); virtual ~DX11_Hook();
void start_hook();
void resetRenderState(); void resetRenderState();
void prepareForOverlay(IDXGISwapChain* pSwapChain); void prepareForOverlay(IDXGISwapChain* pSwapChain);
@ -43,7 +42,8 @@ private:
//decltype(D3D11CreateDeviceAndSwapChain)* D3D11CreateDeviceAndSwapChain; //decltype(D3D11CreateDeviceAndSwapChain)* D3D11CreateDeviceAndSwapChain;
public: public:
static void Create(); // Initialize DX11 Hook. bool start_hook();
static DX11_Hook* Inst();
void loadFunctions(ID3D11Device *pDevice, IDXGISwapChain *pSwapChain); void loadFunctions(ID3D11Device *pDevice, IDXGISwapChain *pSwapChain);
}; };

View file

@ -9,15 +9,14 @@
#include <impls/imgui_impl_win32.h> #include <impls/imgui_impl_win32.h>
#include <impls/imgui_impl_dx12.h> #include <impls/imgui_impl_dx12.h>
// This is created by DX12_Hook::Create, and deleted by the Hook_Manager if not used bool DX12_Hook::start_hook()
static DX12_Hook* hook;
void DX12_Hook::start_hook()
{ {
if (!_hooked) if (!_hooked)
{ {
PRINT_DEBUG("Hooked DirectX 12\n"); PRINT_DEBUG("Hooked DirectX 12\n");
return false;
} }
return true;
} }
void DX12_Hook::resetRenderState() void DX12_Hook::resetRenderState()

View file

@ -24,7 +24,6 @@ private:
DX12_Hook(); DX12_Hook();
virtual ~DX12_Hook(); virtual ~DX12_Hook();
void start_hook();
void resetRenderState(); void resetRenderState();
void prepareForOverlay(IDXGISwapChain* pSwapChain); void prepareForOverlay(IDXGISwapChain* pSwapChain);
@ -43,7 +42,8 @@ private:
//decltype(D3D12CreateDevice)* D3D12CreateDevice; //decltype(D3D12CreateDevice)* D3D12CreateDevice;
public: public:
static void Create(); // Initialize DX11 Hook. bool start_hook();
static DX12_Hook* Inst();
void loadFunctions(ID3D12Device *pDevice, IDXGISwapChain *pSwapChain); void loadFunctions(ID3D12Device *pDevice, IDXGISwapChain *pSwapChain);
}; };

View file

@ -18,7 +18,7 @@ static DX9_Hook* hook;
///////// ///////// ///////// /////////
////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////
void DX9_Hook::start_hook() bool DX9_Hook::start_hook()
{ {
if (!_hooked) if (!_hooked)
{ {
@ -56,11 +56,13 @@ void DX9_Hook::start_hook()
else else
{ {
PRINT_DEBUG("Failed to DirectX 9\n"); PRINT_DEBUG("Failed to DirectX 9\n");
return false;
} }
if(pDeviceEx)pDeviceEx->Release(); if(pDeviceEx)pDeviceEx->Release();
if(pD3D)pD3D->Release(); if(pD3D)pD3D->Release();
} }
return true;
} }
void DX9_Hook::resetRenderState() void DX9_Hook::resetRenderState()

View file

@ -21,7 +21,6 @@ private:
DX9_Hook(); DX9_Hook();
virtual ~DX9_Hook(); virtual ~DX9_Hook();
void start_hook();
void resetRenderState(); void resetRenderState();
void prepareForOverlay(IDirect3DDevice9* pDevice); void prepareForOverlay(IDirect3DDevice9* pDevice);
@ -44,7 +43,8 @@ private:
//decltype(Direct3DCreate9Ex)* Direct3DCreate9Ex; //decltype(Direct3DCreate9Ex)* Direct3DCreate9Ex;
public: public:
static void Create(); // Initialize DX9 Hook. bool start_hook();
static DX9_Hook* Inst();
void loadFunctions(IDirect3DDevice9Ex *pDeviceEx); void loadFunctions(IDirect3DDevice9Ex *pDeviceEx);
}; };

View file

@ -34,7 +34,17 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain*
if (pDevice) if (pDevice)
{ {
Hook_Manager::Inst().UnHookAllRendererDetector(); Hook_Manager::Inst().UnHookAllRendererDetector();
DX10_Hook::Create(); DX10_Hook* hook = DX10_Hook::Inst();
if (!hook->start_hook())
{
// Hook failed, start over
delete static_cast<Base_Hook*>(hook);
Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
}
else
{
Hook_Manager::Inst().AddHook(hook);
}
} }
else else
{ {
@ -42,15 +52,31 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain*
if (pDevice) if (pDevice)
{ {
Hook_Manager::Inst().UnHookAllRendererDetector(); Hook_Manager::Inst().UnHookAllRendererDetector();
DX11_Hook::Create(); DX11_Hook* hook = DX11_Hook::Inst();
if (!hook->start_hook())
{
// Hook failed, start over
delete static_cast<Base_Hook*>(hook);
Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
}
else
{
Hook_Manager::Inst().AddHook(hook);
}
} }
else else
{ {
_this->GetDevice(__uuidof(ID3D12Device), (void**)& pDevice); _this->GetDevice(__uuidof(ID3D12Device), (void**)& pDevice);
if (pDevice) DX12_Hook* hook = DX12_Hook::Inst();
if (!hook->start_hook())
{ {
Hook_Manager::Inst().UnHookAllRendererDetector(); // Hook failed, start over
DX12_Hook::Create(); delete static_cast<Base_Hook*>(hook);
Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
}
else
{
Hook_Manager::Inst().AddHook(hook);
} }
} }
} }
@ -62,28 +88,60 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain*
HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion) HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion)
{ {
Hook_Manager::Inst().UnHookAllRendererDetector(); Hook_Manager::Inst().UnHookAllRendererDetector();
DX9_Hook::Create(); DX9_Hook* hook = DX9_Hook::Inst();
if (!hook->start_hook())
{
// Hook failed, start over
delete static_cast<Base_Hook*>(hook);
Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
}
else
{
Hook_Manager::Inst().AddHook(hook);
}
return (_this->*_IDirect3DDevice9_Present)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion); return (_this->*_IDirect3DDevice9_Present)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion);
} }
HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags) HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags)
{ {
Hook_Manager::Inst().UnHookAllRendererDetector(); Hook_Manager::Inst().UnHookAllRendererDetector();
DX9_Hook::Create(); DX9_Hook* hook = DX9_Hook::Inst();
if (!hook->start_hook())
{
// Hook failed, start over
delete static_cast<Base_Hook*>(hook);
Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
}
else
{
Hook_Manager::Inst().AddHook(hook);
}
return (_this->*_IDirect3DDevice9Ex_PresentEx)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags); return (_this->*_IDirect3DDevice9Ex_PresentEx)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags);
} }
BOOL WINAPI Hook_Manager::MywglMakeCurrent(HDC hDC, HGLRC hGLRC) BOOL WINAPI Hook_Manager::MywglMakeCurrent(HDC hDC, HGLRC hGLRC)
{ {
Hook_Manager::Inst().UnHookAllRendererDetector(); Hook_Manager::Inst().UnHookAllRendererDetector();
OpenGL_Hook::Create(); OpenGL_Hook* hook = OpenGL_Hook::Inst();
return _wglMakeCurrent(hDC, hGLRC);; if (!hook->start_hook())
{
// Hook failed, start over
delete static_cast<Base_Hook*>(hook);
Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
}
else
{
Hook_Manager::Inst().AddHook(hook);
}
return _wglMakeCurrent(hDC, hGLRC);
} }
void Hook_Manager::HookDXGIPresent() void Hook_Manager::HookDXGIPresent()
{ {
if (!_dxgi_hooked) if (!_dxgi_hooked)
{ {
_dxgi_hooked = true;
rendererdetect_hook->BeginHook(); rendererdetect_hook->BeginHook();
rendererdetect_hook->HookFuncs( rendererdetect_hook->HookFuncs(
@ -98,6 +156,7 @@ void Hook_Manager::HookDX9Present()
{ {
if (!_dx9_hooked) if (!_dx9_hooked)
{ {
_dx9_hooked = true;
rendererdetect_hook->BeginHook(); rendererdetect_hook->BeginHook();
rendererdetect_hook->HookFuncs( rendererdetect_hook->HookFuncs(
@ -113,6 +172,7 @@ void Hook_Manager::HookwglMakeCurrent()
{ {
if (!_ogl_hooked) if (!_ogl_hooked)
{ {
_ogl_hooked = true;
rendererdetect_hook->BeginHook(); rendererdetect_hook->BeginHook();
rendererdetect_hook->HookFuncs( rendererdetect_hook->HookFuncs(
@ -238,7 +298,9 @@ void Hook_Manager::hook_dx12()
{ {
if (!_dxgi_hooked && !_renderer_found) if (!_dxgi_hooked && !_renderer_found)
{ {
DX12_Hook::Create(); DX12_Hook* hook = DX12_Hook::Inst();
hook->start_hook(); // TODO: Prints to error log about DX12 Implementation status
delete static_cast<Base_Hook*>(hook);
} }
} }
@ -350,6 +412,7 @@ void Hook_Manager::UnHookAllRendererDetector()
delete rendererdetect_hook; delete rendererdetect_hook;
rendererdetect_hook = nullptr; rendererdetect_hook = nullptr;
} }
_loadlibrary_hooked = false;
_ogl_hooked = false; _ogl_hooked = false;
#ifdef STEAM_WIN32 #ifdef STEAM_WIN32

View file

@ -38,9 +38,9 @@ protected:
Hook_Manager(); Hook_Manager();
virtual ~Hook_Manager(); virtual ~Hook_Manager();
void UnHookAllRendererDetector();
void UnHookAllRendererDetector();
// Setup opengl device
void hook_opengl(); void hook_opengl();
void HookLoadLibrary(); void HookLoadLibrary();
@ -52,13 +52,21 @@ protected:
bool _dx9_hooked; // DX9 Present and PresentEx Hooked ? bool _dx9_hooked; // DX9 Present and PresentEx Hooked ?
bool _dxgi_hooked; // DXGI Present is hooked ? (DX10, DX11, DX12) bool _dxgi_hooked; // DXGI Present is hooked ? (DX10, DX11, DX12)
// DXGIPresent will be used to detect if DX10, DX11 or DX12 should be used for overlay
void HookDXGIPresent(); void HookDXGIPresent();
// DX9 Present and PresentEx will be used to detect if DX9 should be used for overlay
void HookDX9Present(); void HookDX9Present();
// wglMakeCurrent will be used to detect if OpenGL3 should be used for overlay
void HookwglMakeCurrent(); void HookwglMakeCurrent();
// Setup DX9 Device and get vtable
void hook_dx9(); void hook_dx9();
// Setup DX10 Device and get vtable
void hook_dx10(); void hook_dx10();
// Setup DX11 Device and get vtable
void hook_dx11(); void hook_dx11();
// Setup DX12 Device and get vtable
void hook_dx12(); void hook_dx12();
void create_hookA(const char* libname); void create_hookA(const char* libname);
void create_hookW(const wchar_t* libname); void create_hookW(const wchar_t* libname);
@ -66,9 +74,12 @@ protected:
static HMODULE WINAPI MyLoadLibraryW(LPCWSTR lpLibFileName); static HMODULE WINAPI MyLoadLibraryW(LPCWSTR lpLibFileName);
static HMODULE WINAPI MyLoadLibraryExA(LPCTSTR lpLibFileName, HANDLE hFile, DWORD dwFlags); static HMODULE WINAPI MyLoadLibraryExA(LPCTSTR lpLibFileName, HANDLE hFile, DWORD dwFlags);
static HMODULE WINAPI MyLoadLibraryExW(LPCWSTR lpLibFileName, HANDLE hFile, DWORD dwFlags); static HMODULE WINAPI MyLoadLibraryExW(LPCWSTR lpLibFileName, HANDLE hFile, DWORD dwFlags);
// If this is called, then DX10, DX11 or DX12 will be used to render overlay
static HRESULT STDMETHODCALLTYPE MyIDXGISwapChain_Present(IDXGISwapChain* _this, UINT SyncInterval, UINT Flags); static HRESULT STDMETHODCALLTYPE MyIDXGISwapChain_Present(IDXGISwapChain* _this, UINT SyncInterval, UINT Flags);
// If any of theses is called, then DX9 will be used to render overlay
static HRESULT STDMETHODCALLTYPE MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion); static HRESULT STDMETHODCALLTYPE MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion);
static HRESULT STDMETHODCALLTYPE MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags); static HRESULT STDMETHODCALLTYPE MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags);
// If this is called, then OpenGL 3 will be used to render overlay
static BOOL WINAPI MywglMakeCurrent(HDC hDC, HGLRC hGLRC); static BOOL WINAPI MywglMakeCurrent(HDC hDC, HGLRC hGLRC);
public: public:

View file

@ -13,10 +13,9 @@
#include "steam_overlay.h" #include "steam_overlay.h"
// This is created by OpenGL_Hook::Create, and deleted by the Hook_Manager if not used OpenGL_Hook* OpenGL_Hook::_inst = nullptr;
static OpenGL_Hook* hook;
void OpenGL_Hook::start_hook() bool OpenGL_Hook::start_hook()
{ {
if (!_hooked) if (!_hooked)
{ {
@ -40,8 +39,10 @@ void OpenGL_Hook::start_hook()
PRINT_DEBUG("Failed to hook OpenGL\n"); PRINT_DEBUG("Failed to hook OpenGL\n");
/* Problem: glewInit failed, something is seriously wrong. */ /* Problem: glewInit failed, something is seriously wrong. */
PRINT_DEBUG("Error: %s\n", glewGetErrorString(err)); PRINT_DEBUG("Error: %s\n", glewGetErrorString(err));
return false;
} }
} }
return true;
} }
void OpenGL_Hook::resetRenderState() void OpenGL_Hook::resetRenderState()
@ -106,8 +107,8 @@ void OpenGL_Hook::prepareForOverlay(HDC hDC)
///////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////
BOOL WINAPI OpenGL_Hook::MywglSwapBuffers(HDC hDC) BOOL WINAPI OpenGL_Hook::MywglSwapBuffers(HDC hDC)
{ {
hook->prepareForOverlay(hDC); OpenGL_Hook::Inst()->prepareForOverlay(hDC);
return hook->wglSwapBuffers(hDC); return OpenGL_Hook::Inst()->wglSwapBuffers(hDC);
} }
OpenGL_Hook::OpenGL_Hook(): OpenGL_Hook::OpenGL_Hook():
@ -138,18 +139,15 @@ OpenGL_Hook::~OpenGL_Hook()
ImGui::DestroyContext(); ImGui::DestroyContext();
} }
hook = nullptr; _inst = nullptr;
} }
void OpenGL_Hook::Create() OpenGL_Hook* OpenGL_Hook::Inst()
{ {
if (hook == nullptr) if (_inst == nullptr)
{ _inst = new OpenGL_Hook;
hook = new OpenGL_Hook;
hook->start_hook(); return _inst;
// Register the hook to the Hook Manager
Hook_Manager::Inst().AddHook(hook);
}
} }
#endif//NO_OVERLAY #endif//NO_OVERLAY

View file

@ -13,6 +13,8 @@ public:
//using wglMakeCurrent_t = BOOL(WINAPI*)(HDC, HGLRC); //using wglMakeCurrent_t = BOOL(WINAPI*)(HDC, HGLRC);
private: private:
static OpenGL_Hook* _inst;
// Variables // Variables
bool initialized; bool initialized;
@ -20,7 +22,6 @@ private:
OpenGL_Hook(); OpenGL_Hook();
virtual ~OpenGL_Hook(); virtual ~OpenGL_Hook();
void start_hook();
void resetRenderState(); void resetRenderState();
void prepareForOverlay(HDC hDC); void prepareForOverlay(HDC hDC);
@ -35,7 +36,8 @@ private:
//wglMakeCurrent_t wglMakeCurrent; //wglMakeCurrent_t wglMakeCurrent;
public: public:
static void Create(); // Initialize OGL Hook. bool start_hook();
inline static OpenGL_Hook* Inst();
}; };
#endif//NO_OVERLAY #endif//NO_OVERLAY