compilation fails for Python 3.11 because python initialization api has been deprecated.

[sandeep-gh]

I know I am jumping a few steps ahead since 3.11 is still in beta.

The deprecated info are here: https://bugs.python.org/issue44113

I tried following which seems to compile fine. Will report if the execution doesn't fail.

diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c
index 188c492..8ce1194 100644
--- a/src/python/nxt_python.c
+++ b/src/python/nxt_python.c
@@ -118,6 +118,11 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
             size = (len + 1) * sizeof(wchar_t);
         }
 
+        PyConfig config;
+        PyConfig_InitIsolatedConfig(&config);
+        Py_InitializeFromConfig(&config);
+        PyConfig_Clear(&config);
+
         nxt_py_home = nxt_malloc(size);
         if (nxt_slow_path(nxt_py_home == NULL)) {
             nxt_alert(task, "Failed to allocate memory");
@@ -125,13 +130,13 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
         }
 
         if (pep405) {
-            mbstowcs(nxt_py_home, c->home, len);
-            mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
-            Py_SetProgramName(nxt_py_home);
+            //mbstowcs(nxt_py_home, c->home, len);
+            //mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
+            //Py_SetProgramName(nxt_py_home);
 
         } else {
-            mbstowcs(nxt_py_home, c->home, len + 1);
-            Py_SetPythonHome(nxt_py_home);
+            //mbstowcs(nxt_py_home, c->home, len + 1);
+            //Py_SetPythonHome(nxt_py_home);
         }
 
 #else

[tippexs]

Hi @sandeep-gh thanks for bringing that to our attention. We are always looking towards new versions / implementations.

Appreciate you shared the Link to the Python discussion. As the deprecation starts with version 3.11, which is currently scheduled for October 2022 I believe, but we still need to support 3.8 (EoL 2024), 3.9 (EoL 2025) and 3.10 (EoL 2026) at the same time we will have a look into the module to make the configuration depended from the Python version in use. Reference: https://docs.python.org/3/c-api/apiabiversion.html

Thanks for sharing the diff is a first start! Will keep this issue updated.

[leiserfg]

Python 3.11 is out, any update on this?

[ac000]

@sandeep-gh Thanks for the patch.

diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c
index 188c492..8ce1194 100644
--- a/src/python/nxt_python.c
+++ b/src/python/nxt_python.c
@@ -118,6 +118,11 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
             size = (len + 1) * sizeof(wchar_t);
         }
 
+        PyConfig config;
+        PyConfig_InitIsolatedConfig(&config);
+        Py_InitializeFromConfig(&config);
+        PyConfig_Clear(&config);
+

OK, will need to check what these do and if they can be used with any previous version of Python and guard them behind some #if/#endif based on Python version.

@@ -125,13 +130,13 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
         }
 
         if (pep405) {
-            mbstowcs(nxt_py_home, c->home, len);
-            mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
-            Py_SetProgramName(nxt_py_home);
+            //mbstowcs(nxt_py_home, c->home, len);
+            //mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
+            //Py_SetProgramName(nxt_py_home);
 
         } else {
-            mbstowcs(nxt_py_home, c->home, len + 1);
-            Py_SetPythonHome(nxt_py_home);
+            //mbstowcs(nxt_py_home, c->home, len + 1);
+            //Py_SetPythonHome(nxt_py_home);
         }
 
 #else

~~This hunk looks superfluous. Left over from debugging?~~

Forget that comment...

[ac000]

Here's a patch that should work for both old & new pythons. It passes our internal testing but having it tested in real world scenarios would be good. Grab the below patch or grab it here.

diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c
index 37204051..3c4e2f79 100644
--- a/src/python/nxt_python.c
+++ b/src/python/nxt_python.c
@@ -22,6 +22,10 @@ typedef struct {
 } nxt_py_thread_info_t;
 
 
+#if PY_MAJOR_VERSION == 3
+static nxt_int_t nxt_python3_init_config(nxt_int_t pep405);
+#endif
+
 static nxt_int_t nxt_python_start(nxt_task_t *task,
     nxt_process_data_t *data);
 static nxt_int_t nxt_python_set_target(nxt_task_t *task,
@@ -64,6 +68,62 @@ static nxt_py_thread_info_t  *nxt_py_threads;
 static nxt_python_proto_t    nxt_py_proto;
 
 
+#if PY_MAJOR_VERSION == 3
+
+static nxt_int_t
+nxt_python3_init_config(nxt_int_t pep405)
+{
+#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 8)
+    PyStatus  status;
+    PyConfig  config;
+
+    PyConfig_InitIsolatedConfig(&config);
+#endif
+
+    if (pep405) {
+#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 8)
+        status = PyConfig_SetString(&config, &config.program_name,
+                                    nxt_py_home);
+        if (PyStatus_Exception(status)) {
+            goto pyinit_exception;
+        }
+#else
+        Py_SetProgramName(nxt_py_home);
+#endif
+
+    } else {
+#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 8)
+        status =PyConfig_SetString(&config, &config.home, nxt_py_home);
+        if (PyStatus_Exception(status)) {
+            goto pyinit_exception;
+        }
+#else
+        Py_SetPythonHome(nxt_py_home);
+#endif
+    }
+
+#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 8)
+    status = Py_InitializeFromConfig(&config);
+    if (PyStatus_Exception(status)) {
+        goto pyinit_exception;
+    }
+    PyConfig_Clear(&config);
+#endif
+
+    return NXT_OK;
+
+#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 8)
+pyinit_exception:
+
+    PyConfig_Clear(&config);
+
+    return NXT_ERROR;
+#endif
+}
+
+#endif
+
+
 static nxt_int_t
 nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
 {
@@ -127,11 +187,15 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
         if (pep405) {
             mbstowcs(nxt_py_home, c->home, len);
             mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
-            Py_SetProgramName(nxt_py_home);
 
         } else {
             mbstowcs(nxt_py_home, c->home, len + 1);
-            Py_SetPythonHome(nxt_py_home);
+        }
+
+        ret = nxt_python3_init_config(pep405);
+        if (nxt_slow_path(ret == NXT_ERROR)) {
+            nxt_alert(task, "Failed to initialise config");
+            return NXT_ERROR;
         }
 
 #else

[sandeep-gh]

@ac000, I haven't tested my patch on older version on Python. Thank you for expanding on this. Hope to test the modified/expanded patch sometime in this month.

[mattxtaz]

@ac000 I have tested this patch on FreeBSD with both Python 3.10 and 3.11 and it works on both with no apparent issues. I'm hoping this can make it into the next release version so that I can upgrade to 3.11 properly.

[ac000]

@mattxtaz Thanks for testing!. Yes, the plan is to have it in the next release as we also require it for Fedora 37 support.