[PATCH 2 of 5] SSL: compatibility with opaque ASN1_NUMBER in OpenSSL 4.0

[Maxim Dounin]

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1773535427 -10800
#      Sun Mar 15 03:43:47 2026 +0300
# Node ID 0e0da695221f4b79ce86b415d2ca979a96c24aa4
# Parent  f4b3140bee29158dbd3fe9c82f748d17bb25546a
SSL: compatibility with opaque ASN1_NUMBER in OpenSSL 4.0.

Previously, ASN1_NUMBER was used directly in ngx_ssl_ocsp_create_key(),
which is no longer possible with OpenSSL 4.0 alpha 1.  The fix is to
convert it to BIGNUM with the ASN1_INTEGER_to_BN() function, and then
copy to the buffer with BN_bn2bin().

This introduces an otherwise unneeded memory allocation for BIGNUM, but
seems to be the only valid way to access the data.  Alternative approach
would be to use ASN1_STRING_length() and ASN1_STRING_get0_data(), but
ASN1_STRING_length(3) explicitly says that ASN1_STRING functions should
not be used for ASN1_INTEGER.

diff --git a/src/event/ngx_event_openssl_stapling.c b/src/event/ngx_event_openssl_stapling.c
--- a/src/event/ngx_event_openssl_stapling.c
+++ b/src/event/ngx_event_openssl_stapling.c
@@ -2629,7 +2629,9 @@ ngx_ssl_ocsp_cache_store(ngx_ssl_ocsp_ct
 static ngx_int_t
 ngx_ssl_ocsp_create_key(ngx_ssl_ocsp_ctx_t *ctx)
 {
+    int               n;
     u_char           *p;
+    BIGNUM           *bn;
     ASN1_INTEGER     *serial;
     const X509_NAME  *name;
 
@@ -2655,12 +2657,23 @@ ngx_ssl_ocsp_create_key(ngx_ssl_ocsp_ctx
     p += 20;
 
     serial = X509_get_serialNumber(ctx->cert);
-    if (serial->length > 20) {
-        return NGX_ERROR;
+
+    bn = ASN1_INTEGER_to_BN(serial, NULL);
+    if (bn == NULL) {
+         return NGX_ERROR;
     }
 
-    p = ngx_cpymem(p, serial->data, serial->length);
-    ngx_memzero(p, 20 - serial->length);
+    if (BN_num_bytes(bn) > 20) {
+         BN_free(bn);
+         return NGX_ERROR;
+    }
+
+    n = BN_bn2bin(bn, p);
+    p += n;
+
+    ngx_memzero(p, 20 - n);
+
+    BN_free(bn);
 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ctx->log, 0,
                    "ssl ocsp key %xV", &ctx->key);