Missing defines need for pypy3 with LibreSSL 3.7.x

[orbea]

When building pypy3 7.3.11 on Gentoo the build fails because of several missing defines in LibreSSL.

x86_64-gentoo-linux-musl-gcc -O2 -pipe -Werror=implicit-function-declaration -Werror=implicit-int -fPIC -I/var/tmp/portage/dev-python/pypy3-7.3.11_p1/work/pypy3.9-v7.3.11-src/include/pypy3.9 -c _pypy_openssl.c -o ./_pypy_openssl.o
_pypy_openssl.c: In function '_cffi_const_ERR_LIB_ASYNC':
_pypy_openssl.c:7741:12: error: 'ERR_LIB_ASYNC' undeclared (first use in this function); did you mean 'ERR_LIB_ASN1'?
 7741 |   int n = (ERR_LIB_ASYNC) <= 0;
      |            ^~~~~~~~~~~~~
      |            ERR_LIB_ASN1
_pypy_openssl.c:7741:12: note: each undeclared identifier is reported only once for each function it appears in
_pypy_openssl.c: In function '_cffi_const_ERR_LIB_OSSL_STORE':
_pypy_openssl.c:7958:12: error: 'ERR_LIB_OSSL_STORE' undeclared (first use in this function); did you mean 'ERR_LIB_OSSL_ENCODER'?
 7958 |   int n = (ERR_LIB_OSSL_STORE) <= 0;
      |            ^~~~~~~~~~~~~~~~~~
      |            ERR_LIB_OSSL_ENCODER
_pypy_openssl.c: In function '_cffi_const_ERR_LIB_SM2':
_pypy_openssl.c:8014:12: error: 'ERR_LIB_SM2' undeclared (first use in this function); did you mean 'ERR_LIB_CMP'?
 8014 |   int n = (ERR_LIB_SM2) <= 0;
      |            ^~~~~~~~~~~
      |            ERR_LIB_CMP
_pypy_openssl.c: In function '_cffi_const_SSL3_MT_CHANGE_CIPHER_SPEC':
_pypy_openssl.c:8901:12: error: 'SSL3_MT_CHANGE_CIPHER_SPEC' undeclared (first use in this function); did you mean 'SSL3_RT_CHANGE_CIPHER_SPEC'?
 8901 |   int n = (SSL3_MT_CHANGE_CIPHER_SPEC) <= 0;
      |            ^~~~~~~~~~~~~~~~~~~~~~~~~~
      |            SSL3_RT_CHANGE_CIPHER_SPEC
_pypy_openssl.c: In function '_cffi_const_SSL3_RT_HEADER':
_pypy_openssl.c:8943:12: error: 'SSL3_RT_HEADER' undeclared (first use in this function); did you mean 'SSL3_RT_ALERT'?
 8943 |   int n = (SSL3_RT_HEADER) <= 0;
      |            ^~~~~~~~~~~~~~
      |            SSL3_RT_ALERT
_pypy_openssl.c: In function '_cffi_const_SSL3_RT_INNER_CONTENT_TYPE':
_pypy_openssl.c:8950:12: error: 'SSL3_RT_INNER_CONTENT_TYPE' undeclared (first use in this function); did you mean 'ASN1_R_NO_CONTENT_TYPE'?
 8950 |   int n = (SSL3_RT_INNER_CONTENT_TYPE) <= 0;
      |            ^~~~~~~~~~~~~~~~~~~~~~~~~~
      |            ASN1_R_NO_CONTENT_TYPE

I rebased the OpenBSD patches that still apply

--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
@@ -91,9 +91,7 @@ static const long Cryptography_HAS_LOCKING_CALLBACKS = 0;
 #endif
 
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110
-static const long Cryptography_HAS_OPENSSL_CLEANUP = 0;
-
-void (*OPENSSL_cleanup)(void) = NULL;
+static const long Cryptography_HAS_OPENSSL_CLEANUP = 1;
 
 /* This function has a significantly different signature pre-1.1.0. since it is
  * for testing only, we don't bother to expose it on older OpenSSLs.
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ct.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ct.py
@@ -5,7 +5,7 @@
 from __future__ import absolute_import, division, print_function
 
 INCLUDES = """
-#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER || CRYPTOGRAPHY_IS_LIBRESSL
 #include <openssl/ct.h>
 
 typedef STACK_OF(SCT) Cryptography_STACK_OF_SCT;
@@ -65,7 +65,7 @@ int SCT_set_log_entry_type(SCT *, ct_log_entry_type_t);
 """
 
 CUSTOMIZATIONS = """
-#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER || CRYPTOGRAPHY_IS_LIBRESSL
 static const long Cryptography_HAS_SCT = 1;
 #else
 static const long Cryptography_HAS_SCT = 0;
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/dh.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/dh.py
@@ -110,7 +110,7 @@ int DH_set0_key(DH *dh, BIGNUM *pub_key, BIGNUM *priv_key)
 }
 #endif
 
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 && !CRYPTOGRAPHY_LIBRESSL_27_OR_GREATER
 #ifndef DH_CHECK_Q_NOT_PRIME
 #define DH_CHECK_Q_NOT_PRIME            0x10
 #endif
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
@@ -215,10 +215,11 @@ void (*EVP_MD_do_all_provided)(OSSL_LIB_CTX *, void (*)(EVP_MD *, void *), void
 #endif
 
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_111
-static const long Cryptography_HAS_ONESHOT_EVP_DIGEST_SIGN_VERIFY = 0;
-static const long Cryptography_HAS_RAW_KEY = 0;
 static const long Cryptography_HAS_EVP_DIGESTFINAL_XOF = 0;
 int (*EVP_DigestFinalXOF)(EVP_MD_CTX *, unsigned char *, size_t) = NULL;
+#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x3070000fL
+static const long Cryptography_HAS_RAW_KEY = 0;
+static const long Cryptography_HAS_ONESHOT_EVP_DIGEST_SIGN_VERIFY = 0;
 int (*EVP_DigestSign)(EVP_MD_CTX *, unsigned char *, size_t *,
                       const unsigned char *tbs, size_t) = NULL;
 int (*EVP_DigestVerify)(EVP_MD_CTX *, const unsigned char *, size_t,
@@ -234,6 +235,10 @@ int (*EVP_PKEY_get_raw_public_key)(const EVP_PKEY *, unsigned char *,
 #else
 static const long Cryptography_HAS_ONESHOT_EVP_DIGEST_SIGN_VERIFY = 1;
 static const long Cryptography_HAS_RAW_KEY = 1;
+#endif
+#else
+static const long Cryptography_HAS_ONESHOT_EVP_DIGEST_SIGN_VERIFY = 1;
+static const long Cryptography_HAS_RAW_KEY = 1;
 static const long Cryptography_HAS_EVP_DIGESTFINAL_XOF = 1;
 #endif
 
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/hmac.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/hmac.py
@@ -24,7 +24,7 @@ void Cryptography_HMAC_CTX_free(HMAC_CTX *ctx);
 
 CUSTOMIZATIONS = """
 HMAC_CTX *Cryptography_HMAC_CTX_new(void) {
-#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER || defined(LIBRESSL_VERSION_NUMBER)
     return HMAC_CTX_new();
 #else
     /* This uses OPENSSL_zalloc in 1.1.0, which is malloc + memset */
@@ -36,7 +36,7 @@ HMAC_CTX *Cryptography_HMAC_CTX_new(void) {
 
 
 void Cryptography_HMAC_CTX_free(HMAC_CTX *ctx) {
-#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER || defined(LIBRESSL_VERSION_NUMBER)
     return HMAC_CTX_free(ctx);
 #else
     if (ctx != NULL) {
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ocsp.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ocsp.py
@@ -109,7 +109,7 @@ struct ocsp_basic_response_st {
 };
 #endif
 
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 && LIBRESSL_VERSION_NUMBER < 0x3050000fL
 /* These functions are all taken from ocsp_cl.c in OpenSSL 1.1.0 */
 const OCSP_CERTID *OCSP_SINGLERESP_get0_id(const OCSP_SINGLERESP *single)
 {
@@ -148,7 +148,7 @@ const ASN1_OCTET_STRING *OCSP_resp_get0_signature(const OCSP_BASICRESP *bs)
 }
 #endif
 
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110J
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110J && LIBRESSL_VERSION_NUMBER < 0x3050000fL
 const X509_ALGOR *OCSP_resp_get0_tbs_sigalg(const OCSP_BASICRESP *bs)
 {
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -612,7 +612,7 @@ CUSTOMIZATIONS = """
 
 /* Added in 1.0.2 but we need it in all versions now due to the great
    opaquing. */
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 && !CRYPTOGRAPHY_IS_LIBRESSL
 /* from ssl/ssl_lib.c */
 const SSL_METHOD *SSL_CTX_get_ssl_method(SSL_CTX *ctx) {
     return ctx->method;
@@ -742,8 +742,8 @@ static const long Cryptography_HAS_SET_CERT_CB = 1;
 
 /* In OpenSSL 1.0.2i+ the handling of COMP_METHOD when OPENSSL_NO_COMP was
    changed and we no longer need to typedef void */
-#if (defined(OPENSSL_NO_COMP) && CRYPTOGRAPHY_OPENSSL_LESS_THAN_102I) || \
-    CRYPTOGRAPHY_IS_LIBRESSL
+#if (defined(OPENSSL_NO_COMP) && CRYPTOGRAPHY_OPENSSL_LESS_THAN_102I) && \
+    !CRYPTOGRAPHY_IS_LIBRESSL
 static const long Cryptography_HAS_COMPRESSION = 0;
 typedef void COMP_METHOD;
 #else
@@ -817,8 +817,6 @@ const SSL_METHOD *(*DTLS_client_method)(void) = NULL;
 static const long Cryptography_HAS_GENERIC_DTLS_METHOD = 1;
 #endif
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102
-static const long SSL_OP_NO_DTLSv1 = 0;
-static const long SSL_OP_NO_DTLSv1_2 = 0;
 long (*DTLS_set_link_mtu)(SSL *, long) = NULL;
 long (*DTLS_get_link_min_mtu)(SSL *) = NULL;
 #endif
@@ -924,7 +922,7 @@ static const long Cryptography_HAS_CIPHER_DETAILS = 0;
 static const long Cryptography_HAS_CIPHER_DETAILS = 1;
 #endif
 
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_111
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 && !CRYPTOGRAPHY_IS_LIBRESSL
 static const long Cryptography_HAS_TLSv1_3 = 0;
 static const long SSL_OP_NO_TLSv1_3 = 0;
 static const long SSL_VERIFY_POST_HANDSHAKE = 0;
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509.py
@@ -280,7 +280,7 @@ int X509_get_signature_nid(const X509 *x)
 
 /* Added in 1.0.2 but we need it in all versions now due to the great
    opaquing. */
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 && !defined(LIBRESSL_VERSION_NUMBER)
 /* from x509/x_x509.c */
 int i2d_re_X509_tbs(X509 *x, unsigned char **pp)
 {
@@ -306,7 +306,7 @@ X509_REVOKED *Cryptography_X509_REVOKED_dup(X509_REVOKED *rev) {
 
 /* Added in 1.1.0 but we need it in all versions now due to the great
    opaquing. */
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 && !defined(LIBRESSL_VERSION_NUMBER)
 int i2d_re_X509_REQ_tbs(X509_REQ *req, unsigned char **pp)
 {
     req->req_info->enc.modified = 1;
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509name.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509name.py
@@ -75,7 +75,7 @@ Cryptography_STACK_OF_X509_NAME_ENTRY *sk_X509_NAME_ENTRY_dup(
 """
 
 CUSTOMIZATIONS = """
-#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER || defined(LIBRESSL_VERSION_NUMBER)
 int Cryptography_X509_NAME_ENTRY_set(X509_NAME_ENTRY *ne) {
     return X509_NAME_ENTRY_set(ne);
 }

And made minor adjustments.

--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -606,7 +606,7 @@ void SSL_set_msg_callback(SSL *ssl,
 """
 
 CUSTOMIZATIONS = """
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 && !CRYPTOGRAPHY_IS_LIBRESSL
 #error Python 3.7 requires OpenSSL >= 1.0.2
 #endif
 
@@ -768,7 +768,7 @@ int (*SSL_CTX_get_max_proto_version)(SSL_CTX *ctx) = NULL;
 int (*SSL_get_min_proto_version)(SSL *ssl) = NULL;
 int (*SSL_get_max_proto_version)(SSL *ssl) = NULL;
 #endif
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 && !CRYPTOGRAPHY_IS_LIBRESSL
 int (*SSL_CTX_set_min_proto_version)(SSL_CTX *ctx, int version) = NULL;
 int (*SSL_CTX_set_max_proto_version)(SSL_CTX *ctx, int version) = NULL;
 int (*SSL_set_min_proto_version)(SSL *ssl, int version) = NULL;

It seems trickier to get farther than this, any help would be appreciated.

[stefan11111]

I will comment on this, but keep in mind that I have a limited understanding of the issue. The file lib_pypy/_pypy_openssl.c seems to be a generated file, and it can not be patched by /etc/portage/patches. The macros seem to be defined in lib_pypy/_cffi_ssl/_stdssl/errorcodes.py and lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py. Can these be patched to make pypy3 build?

[orbea]

@stefan11111 There are .py files that would need to be patched, you can grep for the lines.

The OpenBSD ports tree has an example.

https://github.com/openbsd/ports/tree/0ee5747b4e9542dfa70a02a96d03c25605c9ad51/lang/pypy/patches

However that is for 7.3.1 when Gentoo currently has 7.3.11 and 7.3.12 which has since changed enough that the older patches aren't enough.