gpg: Add arg session_algo to pk_decrypt.

* common/kem.c: Move constants to the top.  Add some documentation.
* g10/pkglue.c (pk_encrypt): Add arguments session_key and factor code
out to ...
(do_encrypt_rsa_elg): here,
(do_encrypt_ecdh): and here,
(do_encrypt_kem): and here.
* g10/encrypt.c (write_pubkey_enc): Call with session key algorithm.
--

This makes it easier to review the code.

4 files changed, 180 insertions, 126 deletions

diff --git a/common/kem.c b/common/kem.c
index c5de8b102..7227898d1 100644
--- a/common/kem.c
+++ b/common/kem.c
@@ -23,9 +23,10 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copies of the GNU General Public License
+ * You should have received copies of the GNU General Public License
  * and the GNU Lesser General Public License along with this program;
  * if not, see <https://www.gnu.org/licenses/>.
+ * SPDX-License-Identifier: (LGPL-3.0-or-later OR GPL-2.0-or-later)
  */
 
 #include <config.h>
@@ -35,7 +36,18 @@
 #include <gcrypt.h>
 #include "mischelp.h"
 
+
+/* domSeperation as per *PGP specs. */
+#define KMAC_KEY "OpenPGPCompositeKeyDerivationFunction"
+
+/* customizationString as per *PGP specs. */
+#define KMAC_CUSTOM "KDF"
+
+/* The blocksize used for Keccak by compute_kmac256.  */
 #define KECCAK512_BLOCKSIZE 136
+
+
+
 static gpg_error_t
 compute_kmac256 (void *digest, size_t digestlen,
                  const void *key, size_t keylen,
@@ -163,14 +175,16 @@ gnupg_ecc_kem_kdf (void *kek, size_t kek_len,
   return 0;
 }
 
-
-/* domSeperation */
-#define KMAC_KEY "OpenPGPCompositeKeyDerivationFunction"
-
-/* customizationString */
-#define KMAC_CUSTOM "KDF"
-
-/* Compute KEK by combining two KEMs.  */
+/* Compute KEK by combining two KEMs.  The caller provides a buffer
+ * KEK allocated with size KEK_LEN which will receive the computed
+ * KEK. (ECC_SS, ECC_SS_LEN) is the shared secret of the first key.
+ * (ECC_CT, ECC_CT_LEN) is the ciphertext of the first key.
+ * (MLKEM_SS, ECC_SS_LEN) is the shared secret of the second key.
+ * (MLKEM_CT, MLKEM_CT_LEN) is the ciphertext of the second key.
+ * (FIXEDINFO, FIXEDINFO_LEN) is an octet string used to bind the KEK
+ * to a the key; for PGP we use the concatenation of the session key's
+ * algorithm id and the v5 fingerprint of the key.
+ */
 gpg_error_t
 gnupg_kem_combiner (void *kek, size_t kek_len,
                     const void *ecc_ss, size_t ecc_ss_len,
diff --git a/g10/encrypt.c b/g10/encrypt.c
index f9818622e..8ce6164ce 100644
--- a/g10/encrypt.c
+++ b/g10/encrypt.c
@@ -1138,7 +1138,7 @@ write_pubkey_enc (ctrl_t ctrl,
    * build_packet().  */
   frame = encode_session_key (pk->pubkey_algo, dek,
                               pubkey_nbits (pk->pubkey_algo, pk->pkey));
-  rc = pk_encrypt (pk, frame, enc->data);
+  rc = pk_encrypt (pk, frame, dek->algo, enc->data);
   gcry_mpi_release (frame);
   if (rc)
     log_error ("pubkey_encrypt failed: %s\n", gpg_strerror (rc) );
diff --git a/g10/pkglue.c b/g10/pkglue.c
index 64b5f2a06..52d8c1012 100644
--- a/g10/pkglue.c
+++ b/g10/pkglue.c
@@ -1,6 +1,7 @@
 /* pkglue.c - public key operations glue code
  * Copyright (C) 2000, 2003, 2010 Free Software Foundation, Inc.
  * Copyright (C) 2014 Werner Koch
+ * Copyright (C) 2024 g10 Code GmbH.
  *
  * This file is part of GnuPG.
  *
@@ -16,6 +17,7 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ * SPDX-License-Identifier: GPL-3.0-or-later
  */
 
 #include <config.h>
@@ -415,152 +417,190 @@ pk_verify (pubkey_algo_t pkalgo, gcry_mpi_t hash,
 }
 
 
+/* Core of the encryption for KEM algorithms.  See pk_decrypt for a
+ * description of the arguments.  */
+static gpg_error_t
+do_encrypt_kem (PKT_public_key *pk, gcry_mpi_t data, int seskey_algo,
+                gcry_mpi_t *resarr)
+{
 
+  log_debug ("Implement Kyber encryption\n");
+  return gpg_error (GPG_ERR_NOT_IMPLEMENTED);
+}
 
-/*
- * Emulate our old PK interface here - sometime in the future we might
- * change the internal design to directly fit to libgcrypt.  PK is is
- * the OpenPGP public key packet, DATA is an MPI with the to be
- * encrypted data, and RESARR receives the encrypted data.  RESARRAY
- * is expected to be an two item array which will be filled with newly
- * allocated MPIs.
- */
-gpg_error_t
-pk_encrypt (PKT_public_key *pk, gcry_mpi_t data, gcry_mpi_t *resarr)
+
+/* Core of the encryption for the ECDH algorithms.  See pk_decrypt for
+ * a description of the arguments.  */
+static gpg_error_t
+do_encrypt_ecdh (PKT_public_key *pk, gcry_mpi_t data,  gcry_mpi_t *resarr)
 {
-  pubkey_algo_t algo = pk->pubkey_algo;
   gcry_mpi_t *pkey   = pk->pkey;
   gcry_sexp_t s_ciph = NULL;
   gcry_sexp_t s_data = NULL;
   gcry_sexp_t s_pkey = NULL;
   gpg_error_t err;
+  gcry_mpi_t k = NULL;
+  char *curve = NULL;
+  int with_djb_tweak_flag;
+  gcry_mpi_t public = NULL;
+  gcry_mpi_t result = NULL;
+  byte fp[MAX_FINGERPRINT_LEN];
+  byte *shared = NULL;
+  byte *p;
+  size_t nshared;
+  unsigned int nbits;
+
+  err = pk_ecdh_generate_ephemeral_key (pkey, &k);
+  if (err)
+    goto leave;
 
-  /* Make a sexp from pkey.  */
-  if (algo == PUBKEY_ALGO_ELGAMAL || algo == PUBKEY_ALGO_ELGAMAL_E)
-    {
-      err = gcry_sexp_build (&s_pkey, NULL,
-			    "(public-key(elg(p%m)(g%m)(y%m)))",
-			    pkey[0], pkey[1], pkey[2]);
-      /* Put DATA into a simplified S-expression.  */
-      if (!err)
-        err = gcry_sexp_build (&s_data, NULL, "%m", data);
-    }
-  else if (algo == PUBKEY_ALGO_RSA || algo == PUBKEY_ALGO_RSA_E)
+  curve = openpgp_oid_to_str (pkey[0]);
+  if (!curve)
     {
-      err = gcry_sexp_build (&s_pkey, NULL,
-			    "(public-key(rsa(n%m)(e%m)))",
-			    pkey[0], pkey[1]);
-      /* Put DATA into a simplified S-expression.  */
-      if (!err)
-        err = gcry_sexp_build (&s_data, NULL, "%m", data);
+      err = gpg_error_from_syserror ();
+      goto leave;
     }
-  else if (algo == PUBKEY_ALGO_ECDH)
-    {
-      gcry_mpi_t k;
 
-      err = pk_ecdh_generate_ephemeral_key (pkey, &k);
-      if (!err)
-        {
-          char *curve;
+  with_djb_tweak_flag = openpgp_oid_is_cv25519 (pkey[0]);
 
-          curve = openpgp_oid_to_str (pkey[0]);
-          if (!curve)
-            err = gpg_error_from_syserror ();
-          else
-            {
-              int with_djb_tweak_flag = openpgp_oid_is_cv25519 (pkey[0]);
-
-              /* Now use the ephemeral secret to compute the shared point.  */
-              err = gcry_sexp_build (&s_pkey, NULL,
-                                    with_djb_tweak_flag ?
-                                    "(public-key(ecdh(curve%s)(flags djb-tweak)(q%m)))"
-                                    : "(public-key(ecdh(curve%s)(q%m)))",
-                                    curve, pkey[1]);
-              xfree (curve);
-              /* Put K into a simplified S-expression.  */
-              if (!err)
-                err = gcry_sexp_build (&s_data, NULL, "%m", k);
-            }
-          gcry_mpi_release (k);
-        }
+  /* Now use the ephemeral secret to compute the shared point.  */
+  err = gcry_sexp_build (&s_pkey, NULL,
+                         with_djb_tweak_flag ?
+                         "(public-key(ecdh(curve%s)(flags djb-tweak)(q%m)))"
+                         : "(public-key(ecdh(curve%s)(q%m)))",
+                         curve, pkey[1]);
+  if (err)
+    goto leave;
+
+  /* Put K into a simplified S-expression.  */
+  err = gcry_sexp_build (&s_data, NULL, "%m", k);
+  if (err)
+    goto leave;
+
+  /* Run encryption.  */
+  err = gcry_pk_encrypt (&s_ciph, s_data, s_pkey);
+  if (err)
+    goto leave;
+
+  gcry_sexp_release (s_data); s_data = NULL;
+  gcry_sexp_release (s_pkey); s_pkey = NULL;
+
+
+  /* Get the shared point and the ephemeral public key.  */
+  shared = get_data_from_sexp (s_ciph, "s", &nshared);
+  if (!shared)
+    {
+      err = gpg_error_from_syserror ();
+      goto leave;
     }
-  else if (algo == PUBKEY_ALGO_KYBER)
+  err = sexp_extract_param_sos (s_ciph, "e", &public);
+  gcry_sexp_release (s_ciph); s_ciph = NULL;
+  if (DBG_CRYPTO)
     {
-      log_debug ("Implement Kyber encryption\n");
-      err = gpg_error (GPG_ERR_NOT_IMPLEMENTED);
+      log_debug ("ECDH ephemeral key:");
+      gcry_mpi_dump (public);
+      log_printf ("\n");
     }
-  else
-    err = gpg_error (GPG_ERR_PUBKEY_ALGO);
 
-  /* Pass it to libgcrypt. */
-  if (!err)
-    err = gcry_pk_encrypt (&s_ciph, s_data, s_pkey);
+  fingerprint_from_pk (pk, fp, NULL);
+
+  p = gcry_mpi_get_opaque (data, &nbits);
+  result = NULL;
+  err = pk_ecdh_encrypt_with_shared_point (shared, nshared, fp, p,
+                                           (nbits+7)/8, pkey, &result);
+  if (err)
+    goto leave;
+
+  resarr[0] = public; public = NULL;
+  resarr[1] = result; result = NULL;
 
+ leave:
+  gcry_mpi_release (public);
+  gcry_mpi_release (result);
+  xfree (shared);
+  gcry_sexp_release (s_ciph);
   gcry_sexp_release (s_data);
   gcry_sexp_release (s_pkey);
+  xfree (curve);
+  gcry_mpi_release (k);
+  return err;
+}
+
+
+/* Core of the encryption for RSA and Elgamal algorithms.  See
+ * pk_decrypt for a description of the arguments.  */
+static gpg_error_t
+do_encrypt_rsa_elg (PKT_public_key *pk, gcry_mpi_t data, gcry_mpi_t *resarr)
+{
+  pubkey_algo_t algo = pk->pubkey_algo;
+  gcry_mpi_t *pkey   = pk->pkey;
+  gcry_sexp_t s_ciph = NULL;
+  gcry_sexp_t s_data = NULL;
+  gcry_sexp_t s_pkey = NULL;
+  gpg_error_t err;
 
+  if (algo == PUBKEY_ALGO_ELGAMAL || algo == PUBKEY_ALGO_ELGAMAL_E)
+    err = gcry_sexp_build (&s_pkey, NULL,
+                           "(public-key(elg(p%m)(g%m)(y%m)))",
+                           pkey[0], pkey[1], pkey[2]);
+  else
+    err = gcry_sexp_build (&s_pkey, NULL,
+                           "(public-key(rsa(n%m)(e%m)))",
+                           pkey[0], pkey[1]);
   if (err)
-    ;
-  else if (algo == PUBKEY_ALGO_ECDH)
-    {
-      gcry_mpi_t public, result;
-      byte fp[MAX_FINGERPRINT_LEN];
-      byte *shared;
-      size_t nshared;
-
-      /* Get the shared point and the ephemeral public key.  */
-      shared = get_data_from_sexp (s_ciph, "s", &nshared);
-      if (!shared)
-        {
-          err = gpg_error_from_syserror ();
-          goto leave;
-        }
-      err = sexp_extract_param_sos (s_ciph, "e", &public);
-      gcry_sexp_release (s_ciph);
-      s_ciph = NULL;
-      if (DBG_CRYPTO)
-        {
-          log_debug ("ECDH ephemeral key:");
-          gcry_mpi_dump (public);
-          log_printf ("\n");
-        }
+    goto leave;
 
-      result = NULL;
-      fingerprint_from_pk (pk, fp, NULL);
+  err = gcry_sexp_build (&s_data, NULL, "%m", data);
+  if (err)
+    goto leave;
 
-      if (!err)
-        {
-          unsigned int nbits;
-          byte *p = gcry_mpi_get_opaque (data, &nbits);
-          err = pk_ecdh_encrypt_with_shared_point (shared, nshared, fp, p,
-                                                  (nbits+7)/8, pkey, &result);
-        }
-      xfree (shared);
-      if (!err)
-        {
-          resarr[0] = public;
-          resarr[1] = result;
-        }
-      else
-        {
-          gcry_mpi_release (public);
-          gcry_mpi_release (result);
-        }
-    }
-  else /* Elgamal or RSA case.  */
-    { /* Fixme: Add better error handling or make gnupg use
-         S-expressions directly.  */
-      resarr[0] = get_mpi_from_sexp (s_ciph, "a", GCRYMPI_FMT_USG);
-      if (!is_RSA (algo))
-        resarr[1] = get_mpi_from_sexp (s_ciph, "b", GCRYMPI_FMT_USG);
-    }
+  err = gcry_pk_encrypt (&s_ciph, s_data, s_pkey);
+  if (err)
+    goto leave;
+
+  gcry_sexp_release (s_data); s_data = NULL;
+  gcry_sexp_release (s_pkey); s_pkey = NULL;
+
+  resarr[0] = get_mpi_from_sexp (s_ciph, "a", GCRYMPI_FMT_USG);
+  if (!is_RSA (algo))
+    resarr[1] = get_mpi_from_sexp (s_ciph, "b", GCRYMPI_FMT_USG);
 
  leave:
+  gcry_sexp_release (s_data);
+  gcry_sexp_release (s_pkey);
   gcry_sexp_release (s_ciph);
   return err;
 }
 
 
+/*
+ * Emulate our old PK interface here - sometime in the future we might
+ * change the internal design to directly fit to libgcrypt.  PK is is
+ * the OpenPGP public key packet, DATA is an MPI with the to be
+ * encrypted data, and RESARR receives the encrypted data.  RESARRAY
+ * is expected to be an two item array which will be filled with newly
+ * allocated MPIs.  SESKEY_ALGO is required for public key algorithms
+ * which do not encode it in DATA.
+ */
+gpg_error_t
+pk_encrypt (PKT_public_key *pk, gcry_mpi_t data, int seskey_algo,
+            gcry_mpi_t *resarr)
+{
+  pubkey_algo_t algo = pk->pubkey_algo;
+
+  if (algo == PUBKEY_ALGO_KYBER)
+    return do_encrypt_kem (pk, data, seskey_algo, resarr);
+  else if (algo == PUBKEY_ALGO_ECDH)
+    return do_encrypt_ecdh (pk, data, resarr);
+  else if (algo == PUBKEY_ALGO_ELGAMAL || algo == PUBKEY_ALGO_ELGAMAL_E)
+    return do_encrypt_rsa_elg (pk, data, resarr);
+  else if (algo == PUBKEY_ALGO_RSA || algo == PUBKEY_ALGO_RSA_E)
+    return do_encrypt_rsa_elg (pk, data, resarr);
+  else
+    return gpg_error (GPG_ERR_PUBKEY_ALGO);
+}
+
+
 /* Check whether SKEY is a suitable secret key. */
 int
 pk_check_secret_key (pubkey_algo_t pkalgo, gcry_mpi_t *skey)
diff --git a/g10/pkglue.h b/g10/pkglue.h
index 609c17b49..2b5c8b143 100644
--- a/g10/pkglue.h
+++ b/g10/pkglue.h
@@ -31,7 +31,7 @@ gpg_error_t sexp_extract_param_sos_nlz (gcry_sexp_t sexp, const char *param,
 
 int pk_verify (pubkey_algo_t algo, gcry_mpi_t hash, gcry_mpi_t *data,
                gcry_mpi_t *pkey);
-gpg_error_t pk_encrypt (PKT_public_key *pk, gcry_mpi_t data,
+gpg_error_t pk_encrypt (PKT_public_key *pk, gcry_mpi_t data, int seskey_algo,
                         gcry_mpi_t *resarr);
 int pk_check_secret_key (pubkey_algo_t algo, gcry_mpi_t *skey);