openssl: add rsa_pkey_new(), rsa_pkey_from_n_e(), rsa_pkey_to_n_e()

Add function to generate an EVP_PKEY for a specific 'n' and 'e', and function
to get 'n' and 'e' values from existing RSA public key. Also add a function to
generate a new RSA key with a specified number of bits.

src/resolve/resolved-dns-dnssec.c

@@ -14,7 +14,7 @@
 #include "sort-util.h"
 #include "string-table.h"
 
-#if PREFER_OPENSSL
+#if PREFER_OPENSSL && OPENSSL_VERSION_MAJOR >= 3
 #  pragma GCC diagnostic push
 #    pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(RSA*, RSA_free, NULL);

src/shared/openssl-util.c

@@ -128,6 +128,149 @@ int rsa_pkey_to_suitable_key_size(
         return 0;
 }
 
+/* Generate RSA public key from provided "n" and "e" values. Note that if "e" is a number (e.g. uint32_t), it
+ * must be provided here big-endian, e.g. wrap it with htobe32(). */
+int rsa_pkey_from_n_e(const void *n, size_t n_size, const void *e, size_t e_size, EVP_PKEY **ret) {
+        _cleanup_(EVP_PKEY_freep) EVP_PKEY *pkey = NULL;
+
+        assert(n);
+        assert(e);
+        assert(ret);
+
+        _cleanup_(EVP_PKEY_CTX_freep) EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+        if (!ctx)
+                return log_oom_debug();
+
+        _cleanup_(BN_freep) BIGNUM *bn_n = BN_bin2bn(n, n_size, NULL);
+        if (!bn_n)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to create BIGNUM for RSA n.");
+
+        _cleanup_(BN_freep) BIGNUM *bn_e = BN_bin2bn(e, e_size, NULL);
+        if (!bn_e)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to create BIGNUM for RSA e.");
+
+#if OPENSSL_VERSION_MAJOR >= 3
+        if (EVP_PKEY_fromdata_init(ctx) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to initialize EVP_PKEY_CTX.");
+
+        _cleanup_(OSSL_PARAM_BLD_freep) OSSL_PARAM_BLD *bld = OSSL_PARAM_BLD_new();
+        if (!bld)
+                return log_oom_debug();
+
+        if (!OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_RSA_N, bn_n))
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to set RSA OSSL_PKEY_PARAM_RSA_N.");
+
+        if (!OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_RSA_E, bn_e))
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to set RSA OSSL_PKEY_PARAM_RSA_E.");
+
+        _cleanup_(OSSL_PARAM_freep) OSSL_PARAM *params = OSSL_PARAM_BLD_to_param(bld);
+        if (!params)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to build RSA OSSL_PARAM.");
+
+        if (EVP_PKEY_fromdata(ctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to create RSA EVP_PKEY.");
+#else
+        _cleanup_(RSA_freep) RSA *rsa_key = RSA_new();
+        if (!rsa_key)
+                return log_oom_debug();
+
+        if (!RSA_set0_key(rsa_key, bn_n, bn_e, NULL))
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to set RSA n/e.");
+        /* rsa_key owns these now, don't free */
+        TAKE_PTR(bn_n);
+        TAKE_PTR(bn_e);
+
+        pkey = EVP_PKEY_new();
+        if (!pkey)
+                return log_oom_debug();
+
+        if (!EVP_PKEY_assign_RSA(pkey, rsa_key))
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to assign RSA key.");
+        /* pkey owns this now, don't free */
+        TAKE_PTR(rsa_key);
+#endif
+
+        *ret = TAKE_PTR(pkey);
+
+        return 0;
+}
+
+/* Get the "n" and "e" values from the pkey. The values are returned in "bin" format, i.e. BN_bn2bin(). */
+int rsa_pkey_to_n_e(
+                const EVP_PKEY *pkey,
+                void **ret_n,
+                size_t *ret_n_size,
+                void **ret_e,
+                size_t *ret_e_size) {
+
+        assert(pkey);
+        assert(ret_n);
+        assert(ret_n_size);
+        assert(ret_e);
+        assert(ret_e_size);
+
+#if OPENSSL_VERSION_MAJOR >= 3
+        _cleanup_(BN_freep) BIGNUM *bn_n = NULL;
+        if (!EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_N, &bn_n))
+                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get RSA n.");
+
+        _cleanup_(BN_freep) BIGNUM *bn_e = NULL;
+        if (!EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_E, &bn_e))
+                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get RSA e.");
+#else
+        const RSA *rsa = EVP_PKEY_get0_RSA((EVP_PKEY*) pkey);
+        if (!rsa)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO),
+                                       "Failed to get RSA key from public key.");
+
+        const BIGNUM *bn_n = RSA_get0_n(rsa);
+        if (!bn_n)
+                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get RSA n.");
+
+        const BIGNUM *bn_e = RSA_get0_e(rsa);
+        if (!bn_e)
+                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get RSA e.");
+#endif
+
+        size_t n_size = BN_num_bytes(bn_n), e_size = BN_num_bytes(bn_e);
+        _cleanup_free_ void *n = malloc(n_size), *e = malloc(e_size);
+        if (!n || !e)
+                return log_oom_debug();
+
+        assert(BN_bn2bin(bn_n, n) == (int) n_size);
+        assert(BN_bn2bin(bn_e, e) == (int) e_size);
+
+        *ret_n = TAKE_PTR(n);
+        *ret_n_size = n_size;
+        *ret_e = TAKE_PTR(e);
+        *ret_e_size = e_size;
+
+        return 0;
+}
+
+/* Generate a new RSA key with the specified number of bits. */
+int rsa_pkey_new(size_t bits, EVP_PKEY **ret) {
+        assert(ret);
+
+        _cleanup_(EVP_PKEY_CTX_freep) EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+        if (!ctx)
+                return log_oom_debug();
+
+        if (EVP_PKEY_keygen_init(ctx) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to initialize EVP_PKEY_CTX.");
+
+        if (EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, (int) bits) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to set RSA bits to %zu.", bits);
+
+        _cleanup_(EVP_PKEY_freep) EVP_PKEY *pkey = NULL;
+        if (EVP_PKEY_keygen(ctx, &pkey) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to generate ECC key.");
+
+        *ret = TAKE_PTR(pkey);
+
+        return 0;
+}
+
 int pubkey_fingerprint(EVP_PKEY *pk, const EVP_MD *md, void **ret, size_t *ret_size) {
         _cleanup_(EVP_MD_CTX_freep) EVP_MD_CTX* m = NULL;
         _cleanup_free_ void *d = NULL, *h = NULL;

src/shared/openssl-util.h

@@ -21,6 +21,7 @@
 #  endif
 #  if OPENSSL_VERSION_MAJOR >= 3
 #    include <openssl/core_names.h>
+#    include <openssl/param_build.h>
 #  endif
 
 DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(X509_NAME*, X509_NAME_free, NULL);
@@ -35,6 +36,13 @@ DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(PKCS7*, PKCS7_free, NULL);
 DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(SSL*, SSL_free, NULL);
 DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(BIO*, BIO_free, NULL);
 DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(EVP_MD_CTX*, EVP_MD_CTX_free, NULL);
+#if OPENSSL_VERSION_MAJOR >= 3
+DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(OSSL_PARAM*, OSSL_PARAM_free, NULL);
+DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(OSSL_PARAM_BLD*, OSSL_PARAM_BLD_free, NULL);
+#else
+DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(EC_KEY*, EC_KEY_free, NULL);
+DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(RSA*, RSA_free, NULL);
+#endif
 
 static inline void sk_X509_free_allp(STACK_OF(X509) **sk) {
         if (!sk || !*sk)
@@ -51,6 +59,12 @@ int rsa_encrypt_bytes(EVP_PKEY *pkey, const void *decrypted_key, size_t decrypte
 
 int rsa_pkey_to_suitable_key_size(EVP_PKEY *pkey, size_t *ret_suitable_key_size);
 
+int rsa_pkey_new(size_t bits, EVP_PKEY **ret);
+
+int rsa_pkey_from_n_e(const void *n, size_t n_size, const void *e, size_t e_size, EVP_PKEY **ret);
+
+int rsa_pkey_to_n_e(const EVP_PKEY *pkey, void **ret_n, size_t *ret_n_size, void **ret_e, size_t *ret_e_size);
+
 int pubkey_fingerprint(EVP_PKEY *pk, const EVP_MD *md, void **ret, size_t *ret_size);
 
 #else