upstream: Convert RSA and ECDSA key to the libcrypto EVP_PKEY API.

DSA remains unconverted as it will be removed within six months.

Based on patches originally from Dmitry Belyavskiy, but significantly
reworked based on feedback from Bob Beck, Joel Sing and especially
Theo Buehler (apologies to anyone I've missed).

ok tb@

OpenBSD-Commit-ID: d098744e89f1dc7e5952a6817bef234eced648b5

1 files changed, 62 insertions, 21 deletions

diff --git a/ssh-pkcs11-client.c b/ssh-pkcs11-client.c
index 5fa8bf02b..b8d1700f0 100644
--- a/ssh-pkcs11-client.c
+++ b/ssh-pkcs11-client.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-pkcs11-client.c,v 1.19 2023/12/18 14:46:56 djm Exp $ */
+/* $OpenBSD: ssh-pkcs11-client.c,v 1.20 2024/08/15 00:51:51 djm Exp $ */
 /*
  * Copyright (c) 2010 Markus Friedl.  All rights reserved.
  * Copyright (c) 2014 Pedro Martelletto. All rights reserved.
@@ -264,14 +264,17 @@ rsa_encrypt(int flen, const u_char *from, u_char *to, RSA *rsa, int padding)
 	debug3_f("signing with PKCS11 provider %s", helper->path);
 	if (padding != RSA_PKCS1_PADDING)
 		goto fail;
-	key = sshkey_new(KEY_UNSPEC);
-	if (key == NULL) {
+	if ((key = sshkey_new(KEY_UNSPEC)) == NULL) {
 		error_f("sshkey_new failed");
 		goto fail;
 	}
+	if ((key->pkey = EVP_PKEY_new()) == NULL ||
+	   EVP_PKEY_set1_RSA(key->pkey, rsa) != 1) {
+		error_f("pkey setup failed");
+		goto fail;
+	}
+
 	key->type = KEY_RSA;
-	RSA_up_ref(rsa);
-	key->rsa = rsa;
 	if ((r = sshkey_to_blob(key, &blob, &blen)) != 0) {
 		error_fr(r, "encode key");
 		goto fail;
@@ -339,21 +342,22 @@ ecdsa_do_sign(const unsigned char *dgst, int dgst_len, const BIGNUM *inv,
 	if ((helper = helper_by_ec(ec)) == NULL || helper->fd == -1)
 		fatal_f("no helper for PKCS11 key");
 	debug3_f("signing with PKCS11 provider %s", helper->path);
-	nid = sshkey_ecdsa_key_to_nid(ec);
-	if (nid < 0) {
-		error_f("couldn't get curve nid");
-		goto fail;
-	}
 
-	key = sshkey_new(KEY_UNSPEC);
-	if (key == NULL) {
+	if ((key = sshkey_new(KEY_UNSPEC)) == NULL) {
 		error_f("sshkey_new failed");
 		goto fail;
 	}
-	key->ecdsa = ec;
+	if ((key->pkey = EVP_PKEY_new()) == NULL ||
+	    EVP_PKEY_set1_EC_KEY(key->pkey, ec) != 1) {
+		error("pkey setup failed");
+		goto fail;
+	}
+	if ((nid = sshkey_ecdsa_pkey_to_nid(key->pkey)) < 0) {
+		error("couldn't get curve nid");
+		goto fail;
+	}
 	key->ecdsa_nid = nid;
 	key->type = KEY_ECDSA;
-	EC_KEY_up_ref(ec);
 
 	if ((r = sshkey_to_blob(key, &blob, &blen)) != 0) {
 		error_fr(r, "encode key");
@@ -408,16 +412,31 @@ ecdsa_do_finish(EC_KEY *ec)
 static void
 wrap_key(struct helper *helper, struct sshkey *k)
 {
+	RSA *rsa = NULL;
+	EC_KEY *ecdsa = NULL;
+
 	debug3_f("wrap %s for provider %s", sshkey_type(k), helper->path);
 	if (k->type == KEY_RSA) {
-		RSA_set_method(k->rsa, helper->rsa_meth);
+		if ((rsa = EVP_PKEY_get1_RSA(k->pkey)) == NULL)
+			fatal_f("no RSA key");
+		if (RSA_set_method(rsa, helper->rsa_meth) != 1)
+			fatal_f("RSA_set_method failed");
 		if (helper->nrsa++ >= INT_MAX)
 			fatal_f("RSA refcount error");
+		if (EVP_PKEY_set1_RSA(k->pkey, rsa) != 1)
+			fatal_f("EVP_PKEY_set1_RSA failed");
+		RSA_free(rsa);
 #if defined(OPENSSL_HAS_ECC) && defined(HAVE_EC_KEY_METHOD_NEW)
 	} else if (k->type == KEY_ECDSA) {
-		EC_KEY_set_method(k->ecdsa, helper->ec_meth);
+		if ((ecdsa = EVP_PKEY_get1_EC_KEY(k->pkey)) == NULL)
+			fatal_f("no ECDSA key");
+		if (EC_KEY_set_method(ecdsa, helper->ec_meth) != 1)
+			fatal_f("EC_KEY_set_method failed");
 		if (helper->nec++ >= INT_MAX)
 			fatal_f("EC refcount error");
+		if (EVP_PKEY_set1_EC_KEY(k->pkey, ecdsa) != 1)
+			fatal_f("EVP_PKEY_set1_EC_KEY failed");
+		EC_KEY_free(ecdsa);
 #endif
 	} else
 		fatal_f("unknown key type");
@@ -437,6 +456,10 @@ pkcs11_make_cert(const struct sshkey *priv,
 	struct helper *helper = NULL;
 	struct sshkey *ret;
 	int r;
+	RSA *rsa_priv = NULL, *rsa_cert = NULL;
+#if defined(OPENSSL_HAS_ECC) && defined(HAVE_EC_KEY_METHOD_NEW)
+	EC_KEY *ec_priv = NULL, *ec_cert = NULL;
+#endif
 
 	debug3_f("private key type %s cert type %s", sshkey_type(priv),
 	    sshkey_type(certpub));
@@ -449,24 +472,42 @@ pkcs11_make_cert(const struct sshkey *priv,
 	}
 	*certprivp = NULL;
 	if (priv->type == KEY_RSA) {
-		if ((helper = helper_by_rsa(priv->rsa)) == NULL ||
+		if ((rsa_priv = EVP_PKEY_get1_RSA(priv->pkey)) == NULL)
+			fatal_f("no RSA pkey");
+		if ((helper = helper_by_rsa(rsa_priv)) == NULL ||
 		    helper->fd == -1)
 			fatal_f("no helper for PKCS11 RSA key");
 		if ((r = sshkey_from_private(priv, &ret)) != 0)
 			fatal_fr(r, "copy key");
-		RSA_set_method(ret->rsa, helper->rsa_meth);
+		if ((rsa_cert = EVP_PKEY_get1_RSA(ret->pkey)) == NULL)
+			fatal_f("no RSA cert pkey");
+		if (RSA_set_method(rsa_cert, helper->rsa_meth) != 1)
+			fatal_f("RSA_set_method failed");
 		if (helper->nrsa++ >= INT_MAX)
 			fatal_f("RSA refcount error");
+		if (EVP_PKEY_set1_RSA(ret->pkey, rsa_cert) != 1)
+			fatal_f("EVP_PKEY_set1_RSA failed");
+		RSA_free(rsa_priv);
+		RSA_free(rsa_cert);
 #if defined(OPENSSL_HAS_ECC) && defined(HAVE_EC_KEY_METHOD_NEW)
 	} else if (priv->type == KEY_ECDSA) {
-		if ((helper = helper_by_ec(priv->ecdsa)) == NULL ||
+		if ((ec_priv = EVP_PKEY_get1_EC_KEY(priv->pkey)) == NULL)
+			fatal_f("no EC pkey");
+		if ((helper = helper_by_ec(ec_priv)) == NULL ||
 		    helper->fd == -1)
 			fatal_f("no helper for PKCS11 EC key");
 		if ((r = sshkey_from_private(priv, &ret)) != 0)
 			fatal_fr(r, "copy key");
-		EC_KEY_set_method(ret->ecdsa, helper->ec_meth);
+		if ((ec_cert = EVP_PKEY_get1_EC_KEY(ret->pkey)) == NULL)
+			fatal_f("no EC cert pkey");
+		if (EC_KEY_set_method(ec_cert, helper->ec_meth) != 1)
+			fatal_f("EC_KEY_set_method failed");
 		if (helper->nec++ >= INT_MAX)
 			fatal_f("EC refcount error");
+		if (EVP_PKEY_set1_EC_KEY(ret->pkey, ec_cert) != 1)
+			fatal_f("EVP_PKEY_set1_EC_KEY failed");
+		EC_KEY_free(ec_priv);
+		EC_KEY_free(ec_cert);
 #endif
 	} else
 		fatal_f("unknown key type %s", sshkey_type(priv));
@@ -485,7 +526,7 @@ pkcs11_make_cert(const struct sshkey *priv,
 static int
 pkcs11_start_helper_methods(struct helper *helper)
 {
-	RSA_METHOD *rsa_meth;
+	RSA_METHOD *rsa_meth = NULL;
 	EC_KEY_METHOD *ec_meth = NULL;
 #if defined(OPENSSL_HAS_ECC) && defined(HAVE_EC_KEY_METHOD_NEW)
 	int (*ec_init)(EC_KEY *key);