Numpy 2.0.0b1 is not supported

[duburcqa]

Numpy 2.0.0 has been released on beta a few days ago and I waited to check how bad the migration would be. It turns out it is not working out of the box but supporting both 1.x and 2.x should be quite manageable. For instance, a one-liner is enough to fix Boost::Python (https://github.com/boostorg/python/issues/431) on 1.84 release, but I think it should be the same on 1.76.

For eigenpy, it is more tricky. I tried to do it myself, and for now I'm stuck here:

/Users/alexis.duburcq/workspace/src/jiminy/eigenpy/src/numpy.cpp:17:10: error: use of undeclared identifier 'PyArray_TypeNumFromName'
  return PyArray_TypeNumFromName(const_cast<char*>(type->tp_name));

Unfortunately, PyArray_TypeNumFromName has been removed with no alternative apparently. See commit. I tried to backport the old implementation but it relies on some hidden global registry. I'm not sure it is possible to get access to it.

My patch so far:

diff --git a/include/eigenpy/numpy-allocator.hpp b/include/eigenpy/numpy-allocator.hpp
index 6165394..58f6415 100644
--- a/include/eigenpy/numpy-allocator.hpp
+++ b/include/eigenpy/numpy-allocator.hpp
@@ -138,7 +138,7 @@ struct numpy_allocator_impl_matrix<Eigen::Ref<MatType, Options, Stride> > {
                         outer_stride = reverse_strides ? mat.innerStride()
                                                        : mat.outerStride();
 
-      const int elsize = call_PyArray_DescrFromType(Scalar_type_code)->elsize;
+      const int elsize = PyDataType_ELSIZE(call_PyArray_DescrFromType(Scalar_type_code));
       npy_intp strides[2] = {elsize * inner_stride, elsize * outer_stride};
 
       PyArrayObject *pyArray = (PyArrayObject *)call_PyArray_New(
@@ -204,7 +204,7 @@ struct numpy_allocator_impl_matrix<
                         outer_stride = reverse_strides ? mat.innerStride()
                                                        : mat.outerStride();
 
-      const int elsize = call_PyArray_DescrFromType(Scalar_type_code)->elsize;
+      const int elsize = PyDataType_ELSIZE(call_PyArray_DescrFromType(Scalar_type_code));
       npy_intp strides[2] = {elsize * inner_stride, elsize * outer_stride};
 
       PyArrayObject *pyArray = (PyArrayObject *)call_PyArray_New(
diff --git a/include/eigenpy/numpy.hpp b/include/eigenpy/numpy.hpp
index 6ab627d..1cbbcf1 100644
--- a/include/eigenpy/numpy.hpp
+++ b/include/eigenpy/numpy.hpp
@@ -16,6 +16,11 @@
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
 #endif
 
+/* Allow compiling against NumPy 1.x and 2.x
+   see: https://github.com/numpy/numpy/blob/afea8fd66f6bdbde855f5aff0b4e73eb0213c646/doc/source/reference/c-api/array.rst#L1224
+*/
+#if NPY_ABI_VERSION < 0x02000000
+#define PyArray_DescrProto PyArray_Descr
+#endif
+
 #include <numpy/ndarrayobject.h>
 #include <numpy/ufuncobject.h>
 
@@ -170,7 +175,7 @@ inline void call_PyArray_InitArrFuncs(PyArray_ArrFuncs* funcs) {
   PyArray_InitArrFuncs(funcs);
 }
 
-inline int call_PyArray_RegisterDataType(PyArray_Descr* dtype) {
+inline int call_PyArray_RegisterDataType(PyArray_DescrProto* dtype) {
   return PyArray_RegisterDataType(dtype);
 }

Nota Bene: C-modules compiled against Numpy 2.x is backward compatible with Numpy 1.23.5 and onward by default, which means that the minimal requirement for Numpy should be bumped up. Python 3.8 would still be supported but no Python 3.6 nor Python 3.7. Both are already EOL so I don't think it is a big issue.

[duburcqa]

I made some progress thanks to numpy team. I'm just having a segfault in one of the unit tests:

diff --git a/include/eigenpy/numpy-allocator.hpp b/include/eigenpy/numpy-allocator.hpp
index 6165394..58f6415 100644
--- a/include/eigenpy/numpy-allocator.hpp
+++ b/include/eigenpy/numpy-allocator.hpp
@@ -138,7 +138,7 @@ struct numpy_allocator_impl_matrix<Eigen::Ref<MatType, Options, Stride> > {
                         outer_stride = reverse_strides ? mat.innerStride()
                                                        : mat.outerStride();
 
-      const int elsize = call_PyArray_DescrFromType(Scalar_type_code)->elsize;
+      const int elsize = PyDataType_ELSIZE(call_PyArray_DescrFromType(Scalar_type_code));
       npy_intp strides[2] = {elsize * inner_stride, elsize * outer_stride};
 
       PyArrayObject *pyArray = (PyArrayObject *)call_PyArray_New(
@@ -204,7 +204,7 @@ struct numpy_allocator_impl_matrix<
                         outer_stride = reverse_strides ? mat.innerStride()
                                                        : mat.outerStride();
 
-      const int elsize = call_PyArray_DescrFromType(Scalar_type_code)->elsize;
+      const int elsize = PyDataType_ELSIZE(call_PyArray_DescrFromType(Scalar_type_code));
       npy_intp strides[2] = {elsize * inner_stride, elsize * outer_stride};
 
       PyArrayObject *pyArray = (PyArrayObject *)call_PyArray_New(
diff --git a/include/eigenpy/numpy.hpp b/include/eigenpy/numpy.hpp
index 6ab627d..1cbbcf1 100644
--- a/include/eigenpy/numpy.hpp
+++ b/include/eigenpy/numpy.hpp
@@ -16,6 +16,11 @@
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
 #endif
 
+// Allow compiling against NumPy 1.x and 2.x
+#if NPY_ABI_VERSION < 0x02000000
+#define PyArray_DescrProto PyArray_Descr
+#endif
+
 #include <numpy/ndarrayobject.h>
 #include <numpy/ufuncobject.h>
 
@@ -170,7 +175,7 @@ inline void call_PyArray_InitArrFuncs(PyArray_ArrFuncs* funcs) {
   PyArray_InitArrFuncs(funcs);
 }
 
-inline int call_PyArray_RegisterDataType(PyArray_Descr* dtype) {
+inline int call_PyArray_RegisterDataType(PyArray_DescrProto* dtype) {
   return PyArray_RegisterDataType(dtype);
 }
 
diff --git a/include/eigenpy/user-type.hpp b/include/eigenpy/user-type.hpp
index e66b2d9..b7eb67b 100644
--- a/include/eigenpy/user-type.hpp
+++ b/include/eigenpy/user-type.hpp
@@ -171,7 +171,7 @@ struct SpecialMethods<T, NPY_USERDEF> {
     char* srcptr = static_cast<char*>(src);
 
     PyArrayObject* py_array = static_cast<PyArrayObject*>(array);
-    PyArray_CopySwapFunc* copyswap = PyArray_DESCR(py_array)->f->copyswap;
+    PyArray_CopySwapFunc* copyswap = PyDataType_GetArrFuncs(PyArray_DESCR(py_array))->copyswap;
 
     for (npy_intp i = 0; i < n; i++) {
       copyswap(dstptr, srcptr, swap, array);
@@ -189,7 +189,7 @@ struct SpecialMethods<T, NPY_USERDEF> {
       return (npy_bool)(value != ZeroValue);
     } else {
       T tmp_value;
-      PyArray_DESCR(py_array)->f->copyswap(
+      PyDataType_GetArrFuncs(PyArray_DESCR(py_array))->copyswap(
           &tmp_value, ip, PyArray_ISBYTESWAPPED(py_array), array);
       return (npy_bool)(tmp_value != ZeroValue);
     }
diff --git a/src/numpy.cpp b/src/numpy.cpp
index 01018ba..c9a4820 100644
--- a/src/numpy.cpp
+++ b/src/numpy.cpp
@@ -2,6 +2,7 @@
  * Copyright 2020-2022 INRIA
  */
 
+
 #include "eigenpy/numpy.hpp"
 
 namespace eigenpy {
@@ -14,7 +15,11 @@ void import_numpy() {
 }
 
 int PyArray_TypeNum(PyTypeObject* type) {
-  return PyArray_TypeNumFromName(const_cast<char*>(type->tp_name));
+  PyArray_Descr * descr = PyArray_DescrFromTypeObject(reinterpret_cast<PyObject*>(type));
+  if (descr == NULL) {
+      return NPY_NOTYPE;
+  }
+  return descr->type_num;
 }
 
 #if defined _WIN32 || defined __CYGWIN__
diff --git a/src/register.cpp b/src/register.cpp
index ec22dec..c3c9e6d 100644
--- a/src/register.cpp
+++ b/src/register.cpp
@@ -52,9 +52,9 @@ int Register::registerNewType(
     throw std::invalid_argument("PyType_Ready fails to initialize input type.");
   }
 
-  PyArray_Descr* descr_ptr =
-      new PyArray_Descr(*call_PyArray_DescrFromType(NPY_OBJECT));
-  PyArray_Descr& descr = *descr_ptr;
+  PyArray_DescrProto* descr_ptr = new PyArray_DescrProto(
+    *reinterpret_cast<PyArray_DescrProto*>(call_PyArray_DescrFromType(NPY_OBJECT)));
+  PyArray_DescrProto& descr = *descr_ptr;
   descr.typeobj = py_type_ptr;
   descr.kind = 'V';
   descr.byteorder = '=';

[duburcqa]

Done ! Opening a PR.

[jcarpent]

Solve via #448