pybind11 icon indicating copy to clipboard operation
pybind11 copied to clipboard

[BUG]: Thread heap collapse when compile with option /MTd on Windows msvc

Open MichaelCache opened this issue 3 years ago • 3 comments

Required prerequisites

  • [X] Make sure you've read the documentation. Your issue may be addressed there.
  • [X] Search the issue tracker and Discussions to verify that this hasn't already been reported. +1 or comment there if it has.
  • [ ] Consider asking first in the Gitter chat room or in a Discussion.

Problem description

Build two python wrapper package by pybind11 version 2.9.2, and with msvc compiler option /MTd instead of /MDd on Windows. Then import these two packages will cause thread heap collapse when python exit. pic

Demo code to reproduce this runtime error.

Some clue:

  1. This runtime error only exists debug build on Windows. Release is OK.
  2. It's ok if switch pybind11 back to 2.5.0 or earlier. The key difference is in pybind11\include\pybind11\detail\class.h::make_default_metaclass(), line 269, which is type->tp_dealloc = pybind11_meta_dealloc. Add a dealloctor to PyTypeObject.
  3. I tried enable macro WITH_THREAD, it failed neither.

Reproducible example code

// C++ code, two simple class
class Test
{
public:
    Test() = default;
    ~Test() = default;
};

class Test2
{
public:
    Test2() = default;
    ~Test2() = default;
};

// bind code, bind two classes to two python pacakges
// bind.cpp
PYBIND11_MODULE(bind, m)
{
    pybind11::class_<Test>(m, "Test")
        .def(py::init<>());
}

// bind2.cpp
PYBIND11_MODULE(bind2, m)
{
    pybind11::class_<Test2>(m, "Test2")
        .def(py::init<>());
}

// use cmake to generate bind.pyd and bind2.pyd with compile option /MTd instead of /MDd
project(pybind_test)
cmake_minimum_required(VERSION 3.20)
find_package(PythonLibs REQUIRED)
include_directories(pybind11/include)
set(CMAKE_BUILD_TYPE Debug)

set(variables
	CMAKE_CXX_FLAGS_DEBUG
	CMAKE_CXX_FLAGS_RELEASE
	CMAKE_CXX_FLAGS_RELWITHDEBINFO
	CMAKE_CXX_FLAGS_MINSIZEREL
   )

foreach(variable ${variables})
	if(${variable} MATCHES "/MD")
		string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
	endif()
endforeach()

include_directories(${PYTHON_INCLUDE_DIRS})
add_library(bind SHARED bind.cpp)
set_target_properties(bind PROPERTIES SUFFIX ".pyd")
target_link_libraries(bind ${PYTHON_LIBRARIES})

add_library(bind2 SHARED bind2.cpp)
set_target_properties(bind2 PROPERTIES SUFFIX ".pyd")
target_link_libraries(bind2 ${PYTHON_LIBRARIES})

add_custom_command(OUTPUT output_pyd
   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bind.pyd ${CMAKE_BINARY_DIR}/../pac/bind.pyd
   )
add_custom_command(OUTPUT output_pyd2
   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bind2.pyd ${CMAKE_BINARY_DIR}/../pac/pac2/bind2.pyd
   )
add_custom_target(CopyTask ALL DEPENDS output_pyd bind output_pyd2 bind2)

// when compile finished, reorganize packages like this
pac
├─ bind.pyd
├─ __init__.py
│
└─pac2
   ├─ bind2.pyd
   └─ __init__.py

// content pac/__init__.py is
from .bind import *
from .pac2.bind2 import *

// then run this python script will cause runtime error
import pac as p

t = p.Test()
t2 = p.Test2()

MichaelCache avatar Apr 16 '22 10:04 MichaelCache


project(pybind_test)
cmake_minimum_required(VERSION 3.20)

# Make sure we're using the same runtime library setting (MT or MD)
if(MSVC)
    # Set runtime library for both Debug and Release configurations
    foreach(variable CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS_MINSIZEREL)
        string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
    endforeach()
endif()

find_package(PythonLibs REQUIRED)
include_directories(pybind11/include)

include_directories(${PYTHON_INCLUDE_DIRS})

# Compile bind.pyd
add_library(bind SHARED bind.cpp)
set_target_properties(bind PROPERTIES SUFFIX ".pyd")
target_link_libraries(bind ${PYTHON_LIBRARIES})

# Compile bind2.pyd
add_library(bind2 SHARED bind2.cpp)
set_target_properties(bind2 PROPERTIES SUFFIX ".pyd")
target_link_libraries(bind2 ${PYTHON_LIBRARIES})

# Copy the compiled .pyd files to appropriate locations
add_custom_command(OUTPUT output_pyd
   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bind.pyd ${CMAKE_BINARY_DIR}/../pac/bind.pyd
)

add_custom_command(OUTPUT output_pyd2
   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bind2.pyd ${CMAKE_BINARY_DIR}/../pac/pac2/bind2.pyd
)

add_custom_target(CopyTask ALL DEPENDS output_pyd bind output_pyd2 bind2)

ljluestc avatar Nov 23 '24 22:11 ljluestc


project(pybind_test)
cmake_minimum_required(VERSION 3.20)

# Make sure we're using the same runtime library setting (MT or MD)
if(MSVC)
    # Set runtime library for both Debug and Release configurations
    foreach(variable CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS_MINSIZEREL)
        string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
    endforeach()
endif()

find_package(PythonLibs REQUIRED)
include_directories(pybind11/include)

include_directories(${PYTHON_INCLUDE_DIRS})

# Compile bind.pyd
add_library(bind SHARED bind.cpp)
set_target_properties(bind PROPERTIES SUFFIX ".pyd")
target_link_libraries(bind ${PYTHON_LIBRARIES})

# Compile bind2.pyd
add_library(bind2 SHARED bind2.cpp)
set_target_properties(bind2 PROPERTIES SUFFIX ".pyd")
target_link_libraries(bind2 ${PYTHON_LIBRARIES})

# Copy the compiled .pyd files to appropriate locations
add_custom_command(OUTPUT output_pyd
   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bind.pyd ${CMAKE_BINARY_DIR}/../pac/bind.pyd
)

add_custom_command(OUTPUT output_pyd2
   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bind2.pyd ${CMAKE_BINARY_DIR}/../pac/pac2/bind2.pyd
)

add_custom_target(CopyTask ALL DEPENDS output_pyd bind output_pyd2 bind2)

@ljluestc This cmake file is almost same with my origin cmake file. And I had tried it, getting the same failure. It's not work.

MichaelCache avatar Nov 25 '24 12:11 MichaelCache

I am hitting the same error messages with pybind11 2.13.6 when importing 2 pyd built with MSVC and /MTd.

VA-GS avatar Jan 18 '25 19:01 VA-GS