apps/pkeyutl: Fix checks and documentation regarding -peerkey

Reviewed-by: Hugo Landau <hlandau@devever.net>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25958)

2 files changed, 30 insertions, 17 deletions

diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index 4015d1f721..7762f251be 100644
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -81,7 +81,7 @@ const OPTIONS pkeyutl_options[] = {
     {"verify", OPT_VERIFY, '-', "Verify with public key"},
     {"encrypt", OPT_ENCRYPT, '-', "Encrypt input data with public key"},
     {"decrypt", OPT_DECRYPT, '-', "Decrypt input data with private key"},
-    {"derive", OPT_DERIVE, '-', "Derive shared secret"},
+    {"derive", OPT_DERIVE, '-', "Derive shared secret from own and peer (EC)DH keys"},
     {"decap", OPT_DECAP, '-', "Decapsulate shared secret"},
     {"encap", OPT_ENCAP, '-', "Encapsulate shared secret"},
     OPT_CONFIG_OPTION,
@@ -310,7 +310,11 @@ int pkeyutl_main(int argc, char **argv)
         goto opthelp;
     } else if (peerkey != NULL && pkey_op != EVP_PKEY_OP_DERIVE) {
         BIO_printf(bio_err,
-                   "%s: no peer key given (-peerkey parameter).\n", prog);
+                   "%s: -peerkey option not allowed without -derive.\n", prog);
+        goto opthelp;
+    } else if (peerkey == NULL && pkey_op == EVP_PKEY_OP_DERIVE) {
+        BIO_printf(bio_err,
+                   "%s: missing -peerkey option for -derive operation.\n", prog);
         goto opthelp;
     }
 
@@ -739,9 +743,10 @@ static EVP_PKEY_CTX *init_ctx(const char *kdfalg, int *pkeysize,
 static int setup_peer(EVP_PKEY_CTX *ctx, int peerform, const char *file,
                       ENGINE *e)
 {
+    EVP_PKEY *pkey = EVP_PKEY_CTX_get0_pkey(ctx);
     EVP_PKEY *peer = NULL;
     ENGINE *engine = NULL;
-    int ret;
+    int ret = 1;
 
     if (peerform == FORMAT_ENGINE)
         engine = e;
@@ -750,8 +755,14 @@ static int setup_peer(EVP_PKEY_CTX *ctx, int peerform, const char *file,
         BIO_printf(bio_err, "Error reading peer key %s\n", file);
         return 0;
     }
-
-    ret = EVP_PKEY_derive_set_peer(ctx, peer) > 0;
+    if (strcmp(EVP_PKEY_get0_type_name(peer), EVP_PKEY_get0_type_name(pkey)) != 0) {
+        BIO_printf(bio_err,
+                   "Type of peer public key: %s does not match type of private key: %s\n",
+                   EVP_PKEY_get0_type_name(peer), EVP_PKEY_get0_type_name(pkey));
+        ret = 0;
+    } else {
+        ret = EVP_PKEY_derive_set_peer(ctx, peer) > 0;
+    }
 
     EVP_PKEY_free(peer);
     return ret;
diff --git a/doc/man1/openssl-pkeyutl.pod.in b/doc/man1/openssl-pkeyutl.pod.in
index 3d512d9a39..a10a82b013 100644
--- a/doc/man1/openssl-pkeyutl.pod.in
+++ b/doc/man1/openssl-pkeyutl.pod.in
@@ -18,8 +18,6 @@ B<openssl> B<pkeyutl>
 [B<-inkey> I<filename>|I<uri>]
 [B<-keyform> B<DER>|B<PEM>|B<P12>|B<ENGINE>]
 [B<-passin> I<arg>]
-[B<-peerkey> I<file>]
-[B<-peerform> B<DER>|B<PEM>|B<P12>|B<ENGINE>]
 [B<-pubin>]
 [B<-certin>]
 [B<-rev>]
@@ -29,6 +27,8 @@ B<openssl> B<pkeyutl>
 [B<-encrypt>]
 [B<-decrypt>]
 [B<-derive>]
+[B<-peerkey> I<file>]
+[B<-peerform> B<DER>|B<PEM>|B<P12>|B<ENGINE>]
 [B<-encap>]
 [B<-decap>]
 [B<-kdf> I<algorithm>]
@@ -120,15 +120,6 @@ See L<openssl-format-options(1)> for details.
 The input key password source. For more information about the format of I<arg>
 see L<openssl-passphrase-options(1)>.
 
-=item B<-peerkey> I<file>
-
-The peer key file, used by key derivation (agreement) operations.
-
-=item B<-peerform> B<DER>|B<PEM>|B<P12>|B<ENGINE>
-
-The peer key format; unspecified by default.
-See L<openssl-format-options(1)> for details.
-
 =item B<-pubin>
 
 By default a private key is read from the key input.
@@ -189,7 +180,18 @@ Decrypt the input data using a private key.
 
 =item B<-derive>
 
-Derive a shared secret using the peer key.
+Derive a shared secret using own private (EC)DH key and peer key.
+
+=item B<-peerkey> I<file>
+
+File containing the peer public or private (EC)DH key
+to use with the key derivation (agreement) operation.
+Its type must match the type of the own private key given with B<-inkey>.
+
+=item B<-peerform> B<DER>|B<PEM>|B<P12>|B<ENGINE>
+
+The peer key format; unspecified by default.
+See L<openssl-format-options(1)> for details.
 
 =item B<-encap>