64-bit DLL compiled by G++ failed to load.

[trungnt2910]

I have a 64-bit DLL, with this source code:

#include <iostream>

extern "C"
{
	__declspec(dllexport) __cdecl void Greet()
	{
		std::cout << "Hello World!" << std::endl;
	}

	__declspec(dllexport) __cdecl int addNumbers(int a, int b)
	{
		std::cout << "Adding " << a << " and " << b << "..." << std::endl;
		return a + b;
	}
}

I compiled this using g++, downloadable from here.

This is my g++ version

g++ (MinGW-W64 x86_64-posix-seh, built by Brecht Sanders) 11.1.0 Copyright (C) 2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

This is the command I used:

g++ SampleDll.cpp -o SampleDLL.dll --shared 

I. Attempt to load from file.

I have a C source file (always compiled using g++ Test.c MemoryModule.c -o Test.exe -D_WIN64) and copied the whole LoadFromFile function from the demo to my source file:

The source

void LoadFromFile(void)
{
    addNumberProc addNumber;
    HRSRC resourceInfo;
    DWORD resourceSize;
    LPVOID resourceData;
    TCHAR buffer[100];

    HINSTANCE handle = LoadLibrary(DLL_FILE);
    if (handle == NULL)
        return;

    addNumber = (addNumberProc)GetProcAddress(handle, "addNumbers");
    _tprintf(_T("From file: %d\n"), addNumber(1, 2));

    resourceInfo = FindResource(handle, MAKEINTRESOURCE(VS_VERSION_INFO), RT_VERSION);
    _tprintf(_T("FindResource returned 0x%p\n"), resourceInfo);

    resourceSize = SizeofResource(handle, resourceInfo);
    resourceData = LoadResource(handle, resourceInfo);
    _tprintf(_T("Resource data: %ld bytes at 0x%p\n"), resourceSize, resourceData);

    LoadString(handle, 1, buffer, sizeof(buffer));
    _tprintf(_T("String1: %s\n"), buffer);

    LoadString(handle, 20, buffer, sizeof(buffer));
    _tprintf(_T("String2: %s\n"), buffer);

    FreeLibrary(handle);
}

It loads properly:

Adding 1 and 2...
From file: 3
FindResource returned 0x0000000000000000
Resource data: 0 bytes at 0x0000000000000000
String1:
String2: 

II. Attempt to load from Memory

I also copied the LoadFromMemory and the ReadLibrary function:

The source

void* ReadLibrary(size_t* pSize) {
    size_t read;
    void* result;
    FILE* fp;

    fp = _tfopen(DLL_FILE, _T("rb"));
    if (fp == NULL)
    {
        _tprintf(_T("Can't open DLL file \"%s\"."), DLL_FILE);
        return NULL;
    }

    fseek(fp, 0, SEEK_END);
    *pSize = (size_t)(ftell(fp));
    if (*pSize == 0)
    {
        fclose(fp);
        return NULL;
    }

    result = (unsigned char *)malloc(*pSize);
    if (result == NULL)
    {
        return NULL;
    }

    fseek(fp, 0, SEEK_SET);
    read = fread(result, 1, *pSize, fp);
    fclose(fp);
    if (read != *pSize)
    {
        free(result);
        return NULL;
    }

    return result;
}

void LoadFromMemory(void)
{
    void *data;
    size_t size;
    HMEMORYMODULE handle;
    addNumberProc addNumber;
    HMEMORYRSRC resourceInfo;
    DWORD resourceSize;
    LPVOID resourceData;
    TCHAR buffer[100];

    data = ReadLibrary(&size);
    if (data == NULL)
    {
        return;
    }

    handle = MemoryLoadLibrary(data, size);

    if (handle == NULL)
    {
		_tprintf(_T("%i"), GetLastError());
        _tprintf(_T("Can't load library from memory.\n"));
        goto exit;
    }

    addNumber = (addNumberProc)MemoryGetProcAddress(handle, "addNumbers");
    _tprintf(_T("From memory: %d\n"), addNumber(66, 99));

    resourceInfo = MemoryFindResource(handle, MAKEINTRESOURCE(VS_VERSION_INFO), RT_VERSION);
    _tprintf(_T("MemoryFindResource returned 0x%p\n"), resourceInfo);

    resourceSize = MemorySizeofResource(handle, resourceInfo);
    resourceData = MemoryLoadResource(handle, resourceInfo);
    _tprintf(_T("Memory resource data: %ld bytes at 0x%p\n"), resourceSize, resourceData);

    MemoryLoadString(handle, 1, buffer, sizeof(buffer));
    _tprintf(_T("String1: %s\n"), buffer);

    MemoryLoadString(handle, 20, buffer, sizeof(buffer));
    _tprintf(_T("String2: %s\n"), buffer);

    MemoryFreeLibrary(handle);

exit:
    free(data);
}

This time, the test program failed, with an access violation.

III. Attempt to use C#'s P/Invoke:

It just works, using a basic DllImport attribute.

IV. Attempt to use a translated MemoryModule version, from a managed executable:

Fails, for the same reason.

V. Some more stuff.

After using a C# debugger on the translated version, and various printf debugging on the original library, I found this line of code's the culprit:

            BOOL successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);

The access violation happened right after calling the dll entry point, and not because of a null pointer.

TL;DR

• I have a C++ dll, 64-bit, compiled using G++ on MinGW-w64, which is perfectly valid (P/Invoke-able, and LoadLibrary-able).
• Loading it using MemoryModule gives an Access Violation error when trying to call the dll entry point.
• Any other dll's (Compiled using MSVC (64-bit, or 32-bit), or using the lesser-known Orange C compiler (32-bit only)) works normally on MemoryModule.

I do not have any experience with such low level programming, please help me explain why MemoryModule crashed when calling this entry point. Thanks in advance.