Unable to hook API calls using withdll.exe with notepad.exe but could do it with my own built binaries

[PSR009]

I'm using the latest Detours version and have built it for x64. When I'm using withdll.exe to inject the DLL into the binary (x64) built on my own using Visual Studio 2019 it is working but when I'm injecting the same DLL into notepad.exe (x64), it isn't hooking them at all.

My DLL code

#include "pch.h"
#include <stdio.h>
#include <windows.h>
#include "detours.h"

static HANDLE(WINAPI* TrueCreateFileA)(LPCSTR lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile) = CreateFileA;
static HANDLE(WINAPI* TrueCreateFileW)(LPCWSTR lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile) = CreateFileW;
static BOOL(WINAPI* TrueWriteFile)(HANDLE hFile, LPCVOID lpBuffer, DWORD nNumberOfBytesToWrite, LPDWORD lpNumberOfBytesWritten, LPOVERLAPPED lpOverlapped) = WriteFile;

HANDLE WINAPI HookedCreateFileA(LPCSTR lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile)
{
    printf("Hooked --> CreateFileA\n");
    HANDLE ret = TrueCreateFileA(lpFileName, dwDesiredAccess, dwShareMode, lpSecurityAttributes, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile);
    return ret;
}

HANDLE WINAPI HookedCreateFileW(LPCWSTR lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile)
{
    printf("Hooked --> CreateFileW\n");
    HANDLE ret = TrueCreateFileW(lpFileName, dwDesiredAccess, dwShareMode, lpSecurityAttributes, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile);
    return ret;
}

bool WINAPI HookedWriteFile(HANDLE hFile, LPCVOID lpBuffer, DWORD nNumberOfBytesToWrite, LPDWORD lpNumberOfBytesWritten, LPOVERLAPPED lpOverlapped)
{
    printf("Hooked --> WriteFile\n");
    BOOL ret = TrueWriteFile(hFile, lpBuffer, nNumberOfBytesToWrite, lpNumberOfBytesWritten, lpOverlapped);
    return ret;
}

BOOL WINAPI DllMain(HINSTANCE hinst, DWORD dwReason, LPVOID reserved)
{
    LONG error;
    (void)hinst;
    (void)reserved;

    if (DetourIsHelperProcess()) {
        return TRUE;
    }

    if (dwReason == DLL_PROCESS_ATTACH) {
        DetourRestoreAfterWith();

        printf("Dll_FileOps" DETOURS_STRINGIFY(DETOURS_BITS) ".dll:"
            " Starting.\n");
        fflush(stdout);

        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourAttach(&(PVOID&)TrueCreateFileA, HookedCreateFileA);
        DetourAttach(&(PVOID&)TrueCreateFileW, HookedCreateFileW);
        DetourAttach(&(PVOID&)TrueWriteFile, HookedWriteFile);
        error = DetourTransactionCommit();

        if (error == NO_ERROR) {
            printf("Dll_FileOps" DETOURS_STRINGIFY(DETOURS_BITS) ".dll:"
                " Detoured.\n");
        }
        else {
            printf("Dll_FileOps" DETOURS_STRINGIFY(DETOURS_BITS) ".dll:"
                " Error detouring: %ld\n", error);
        }
    }
    else if (dwReason == DLL_PROCESS_DETACH) {
        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourDetach(&(PVOID&)TrueCreateFileA, HookedCreateFileA);
        DetourDetach(&(PVOID&)TrueCreateFileW, HookedCreateFileW);
        DetourDetach(&(PVOID&)TrueWriteFile, HookedWriteFile);
        error = DetourTransactionCommit();

        printf("Dll_FileOps" DETOURS_STRINGIFY(DETOURS_BITS) ".dll:"
            " Removed (result=%ld).\n", error);
        fflush(stdout);
    }
    return TRUE;
}

DEF file

LIBRARY	Dll_FileOps
EXPORTS
	DetourFinishHelperProcess @1
	HookedCreateFileA @2
	HookedCreateFileW @3
	HookedWriteFile @4

Simple Application used

#include <Windows.h>
#include <string.h>

int main()
{
    // Open a handle to the file
    HANDLE hFile = CreateFile(
        L"C:\\Users\\temppc\\source\\repos\\NewFile.txt",     // Filename
        GENERIC_WRITE,          // Desired access
        FILE_SHARE_READ,        // Share mode
        NULL,                   // Security attributes
        CREATE_NEW,             // Creates a new file, only if it doesn't already exist
        FILE_ATTRIBUTE_NORMAL,  // Flags and attributes
        NULL);                  // Template file handle

    if (hFile == INVALID_HANDLE_VALUE)
    {
        // Failed to open/create file
        return 2;
    }

    // Write data to the file
    LPSTR strText = "Hello World!"; // For C use LPSTR (char*) or LPWSTR (wchar_t*)
    DWORD bytesWritten;
    WriteFile(
        hFile,              // Handle to the file
        strText,            // Buffer to write
        strlen(strText),    // Buffer size
        &bytesWritten,      // Bytes written
        NULL);              // Overlapped

     // Close the handle once we don't need it.
    CloseHandle(hFile);
}

Command-line Output

• Using my application built with Visual Studio 2019

C:\Users\temppc\source\repos\Detours\bin.X64>withdll.exe /d:C:\Users\temppc\source\repos\Dll_FileOps\x64\Debug\Dll_FileOps.dll C:\Users\temppc\source\repos\FileOps\x64\Debug\FileOps.exe
withdll.exe: Starting: `C:\Users\temppc\source\repos\FileOps\x64\Debug\FileOps.exe'
withdll.exe:   with `C:\Users\temppc\source\repos\Dll_FileOps\x64\Debug\Dll_FileOps.dll'
Dll_FileOps64.dll: Starting.
Dll_FileOps64.dll: Detoured.
Hooked --> CreateFileW
Hooked --> WriteFile
Dll_FileOps64.dll: Removed (result=0).

• Using notepad.exe

C:\Users\temppc\source\repos\Detours\bin.X64>withdll.exe /d:C:\Users\temppc\source\repos\Dll_FileOps\x64\Debug\Dll_FileOps.dll C:\Windows\notepad.exe  
withdll.exe: Starting: `C:\Windows\notepad.exe'
withdll.exe:   with `C:\Users\temppc\source\repos\Dll_FileOps\x64\Debug\Dll_FileOps.dll'

When I directly write into the text file opened by withdll.exe and save, it simply exits in the above notepad.exe case without any output. Even if I write my DLL code into the samples example, it is working with my binary but not with notepad.exe. Please tell me if I'm missing anything here. Thank you!

[bgianfo]

Thanks for filing this, can you also add what version of windows you are running on? Example:

$ systeminfo | findstr "Version"
OS Version:                10.0.XXXX N/A Build XXXX

[PSR009]

Windows 10 Pro Version 20H2 OS Build 19042.928

> systeminfo | findstr "Version"
OS Version:                10.0.19042 N/A Build 19042

[mooooonlight]

These days i view open issues and discover this issue is similiar with #230 , because FileOps.exe is a console application, so it prints these logs on console.

Dll_FileOps64.dll: Starting.
Dll_FileOps64.dll: Detoured.
Hooked --> CreateFileW
Hooked --> WriteFile

however notepad.exe is a desktop application, so you cound not found these prints. Try to open cmd.exe, i get these prints too. so i think hook is success, just you can not observe from the print.

.\withdll.exe -d:.\Dll_FileOps.dll cmd.exe
withdll.exe: Starting: `cmd.exe'
withdll.exe:   with `D:\Code\BCM\Detours\Detours\bin.X64\Dll_FileOps.dll'
Dll_FileOps64.dll: Starting.
Dll_FileOps64.dll: Detoured.
Hooked --> CreateFileW
Hooked --> CreateFileW
Hooked --> CreateFileW

[zeltrax00]

@mooooonlight Use OutputDebugString and open DebugView to see log.