Pre-ML-KEM/DSA decoder/encoder refactor

Simplify some decoder/encoder internals to facilitate upcoming support
for ML-KEM and ML-DSA.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/26355)

3 files changed, 47 insertions, 61 deletions

diff --git a/providers/implementations/encode_decode/decode_der2key.c b/providers/implementations/encode_decode/decode_der2key.c
index 0384cb5e89..e33ae49b51 100644
--- a/providers/implementations/encode_decode/decode_der2key.c
+++ b/providers/implementations/encode_decode/decode_der2key.c
@@ -42,8 +42,10 @@ struct der2key_ctx_st;           /* Forward declaration */
 typedef int check_key_fn(void *, struct der2key_ctx_st *ctx);
 typedef void adjust_key_fn(void *, struct der2key_ctx_st *ctx);
 typedef void free_key_fn(void *);
-typedef void *d2i_PKCS8_fn(void **, const unsigned char **, long,
+typedef void *d2i_PKCS8_fn(const unsigned char **, long,
                            struct der2key_ctx_st *);
+typedef void *d2i_PUBKEY_fn(const unsigned char **, long,
+                            struct der2key_ctx_st *);
 struct keytype_desc_st {
     const char *keytype_name;
     const OSSL_DISPATCH *fns; /* Keymgmt (to pilfer functions from) */
@@ -67,7 +69,7 @@ struct keytype_desc_st {
     d2i_of_void *d2i_public_key;  /* From type-specific DER */
     d2i_of_void *d2i_key_params;  /* From type-specific DER */
     d2i_PKCS8_fn *d2i_PKCS8;      /* Wrapped in a PrivateKeyInfo */
-    d2i_of_void *d2i_PUBKEY;      /* Wrapped in a SubjectPublicKeyInfo */
+    d2i_PUBKEY_fn *d2i_PUBKEY;    /* Wrapped in a SubjectPublicKeyInfo */
 
     /*
      * For any key, we may need to check that the key meets expectations.
@@ -216,7 +218,7 @@ static int der2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection,
 
     ctx->selection = selection;
     /*
-     * The caller is allowed to specify 0 as a selection mark, to have the
+     * The caller is allowed to specify 0 as a selection mask, to have the
      * structure and key type guessed.  For type-specific structures, this
      * is not recommended, as some structures are very similar.
      * Note that 0 isn't the same as OSSL_KEYMGMT_SELECT_ALL, as the latter
@@ -240,7 +242,7 @@ static int der2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection,
     if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) {
         derp = der;
         if (ctx->desc->d2i_PKCS8 != NULL) {
-            key = ctx->desc->d2i_PKCS8(NULL, &derp, der_len, ctx);
+            key = ctx->desc->d2i_PKCS8(&derp, der_len, ctx);
             if (ctx->flag_fatal) {
                 ERR_clear_last_mark();
                 goto end;
@@ -256,7 +258,7 @@ static int der2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection,
     if (key == NULL && (selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) {
         derp = der;
         if (ctx->desc->d2i_PUBKEY != NULL)
-            key = ctx->desc->d2i_PUBKEY(NULL, &derp, der_len);
+            key = ctx->desc->d2i_PUBKEY(&derp, der_len, ctx);
         else if (ctx->desc->d2i_public_key != NULL)
             key = ctx->desc->d2i_public_key(NULL, &derp, der_len);
         if (key == NULL && ctx->selection != 0) {
@@ -367,6 +369,14 @@ static int der2key_export_object(void *vctx,
     return 0;
 }
 
+#define D2I_PUBKEY_NOCTX(n, f)                              \
+    static void *                                           \
+    n##_d2i_PUBKEY(const unsigned char **der, long der_len, \
+                   ossl_unused struct der2key_ctx_st *ctx)  \
+    {                                                       \
+        return f(NULL, der, der_len);                       \
+    }
+
 /* ---------------------------------------------------------------------- */
 
 #ifndef OPENSSL_NO_DH
@@ -374,17 +384,18 @@ static int der2key_export_object(void *vctx,
 # define dh_d2i_private_key             NULL
 # define dh_d2i_public_key              NULL
 # define dh_d2i_key_params              (d2i_of_void *)d2i_DHparams
+# define dh_free                        (free_key_fn *)DH_free
+# define dh_check                       NULL
 
-static void *dh_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
+static void *dh_d2i_PKCS8(const unsigned char **der, long der_len,
                           struct der2key_ctx_st *ctx)
 {
     return der2key_decode_p8(der, der_len, ctx,
                              (key_from_pkcs8_t *)ossl_dh_key_from_pkcs8);
 }
 
-# define dh_d2i_PUBKEY                  (d2i_of_void *)ossl_d2i_DH_PUBKEY
-# define dh_free                        (free_key_fn *)DH_free
-# define dh_check                       NULL
+D2I_PUBKEY_NOCTX(dh, ossl_d2i_DH_PUBKEY)
+D2I_PUBKEY_NOCTX(dhx, ossl_d2i_DHx_PUBKEY)
 
 static void dh_adjust(void *key, struct der2key_ctx_st *ctx)
 {
@@ -396,7 +407,6 @@ static void dh_adjust(void *key, struct der2key_ctx_st *ctx)
 # define dhx_d2i_public_key             NULL
 # define dhx_d2i_key_params             (d2i_of_void *)d2i_DHxparams
 # define dhx_d2i_PKCS8                  dh_d2i_PKCS8
-# define dhx_d2i_PUBKEY                 (d2i_of_void *)ossl_d2i_DHx_PUBKEY
 # define dhx_free                       (free_key_fn *)DH_free
 # define dhx_check                      NULL
 # define dhx_adjust                     dh_adjust
@@ -409,17 +419,17 @@ static void dh_adjust(void *key, struct der2key_ctx_st *ctx)
 # define dsa_d2i_private_key            (d2i_of_void *)d2i_DSAPrivateKey
 # define dsa_d2i_public_key             (d2i_of_void *)d2i_DSAPublicKey
 # define dsa_d2i_key_params             (d2i_of_void *)d2i_DSAparams
+# define dsa_free                       (free_key_fn *)DSA_free
+# define dsa_check                      NULL
 
-static void *dsa_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
+static void *dsa_d2i_PKCS8(const unsigned char **der, long der_len,
                            struct der2key_ctx_st *ctx)
 {
     return der2key_decode_p8(der, der_len, ctx,
                              (key_from_pkcs8_t *)ossl_dsa_key_from_pkcs8);
 }
 
-# define dsa_d2i_PUBKEY                 (d2i_of_void *)ossl_d2i_DSA_PUBKEY
-# define dsa_free                       (free_key_fn *)DSA_free
-# define dsa_check                      NULL
+D2I_PUBKEY_NOCTX(dsa, ossl_d2i_DSA_PUBKEY)
 
 static void dsa_adjust(void *key, struct der2key_ctx_st *ctx)
 {
@@ -434,16 +444,16 @@ static void dsa_adjust(void *key, struct der2key_ctx_st *ctx)
 # define ec_d2i_private_key             (d2i_of_void *)d2i_ECPrivateKey
 # define ec_d2i_public_key              NULL
 # define ec_d2i_key_params              (d2i_of_void *)d2i_ECParameters
+# define ec_free                        (free_key_fn *)EC_KEY_free
 
-static void *ec_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
+static void *ec_d2i_PKCS8(const unsigned char **der, long der_len,
                           struct der2key_ctx_st *ctx)
 {
     return der2key_decode_p8(der, der_len, ctx,
                              (key_from_pkcs8_t *)ossl_ec_key_from_pkcs8);
 }
 
-# define ec_d2i_PUBKEY                  (d2i_of_void *)d2i_EC_PUBKEY
-# define ec_free                        (free_key_fn *)EC_KEY_free
+D2I_PUBKEY_NOCTX(ec, d2i_EC_PUBKEY)
 
 static int ec_check(void *key, struct der2key_ctx_st *ctx)
 {
@@ -471,13 +481,18 @@ static void ec_adjust(void *key, struct der2key_ctx_st *ctx)
  * so no d2i functions to be had.
  */
 
-static void *ecx_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
+static void *ecx_d2i_PKCS8(const unsigned char **der, long der_len,
                            struct der2key_ctx_st *ctx)
 {
     return der2key_decode_p8(der, der_len, ctx,
                              (key_from_pkcs8_t *)ossl_ecx_key_from_pkcs8);
 }
 
+D2I_PUBKEY_NOCTX(ed25519, ossl_d2i_ED25519_PUBKEY)
+D2I_PUBKEY_NOCTX(ed448, ossl_d2i_ED448_PUBKEY)
+D2I_PUBKEY_NOCTX(x25519, ossl_d2i_X25519_PUBKEY)
+D2I_PUBKEY_NOCTX(x448, ossl_d2i_X448_PUBKEY)
+
 static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx)
 {
     ossl_ecx_key_set0_libctx(key, PROV_LIBCTX_OF(ctx->provctx));
@@ -488,7 +503,6 @@ static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx)
 #  define ed25519_d2i_public_key         NULL
 #  define ed25519_d2i_key_params         NULL
 #  define ed25519_d2i_PKCS8              ecx_d2i_PKCS8
-#  define ed25519_d2i_PUBKEY             (d2i_of_void *)ossl_d2i_ED25519_PUBKEY
 #  define ed25519_free                   (free_key_fn *)ossl_ecx_key_free
 #  define ed25519_check                  NULL
 #  define ed25519_adjust                 ecx_key_adjust
@@ -498,7 +512,6 @@ static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx)
 #  define ed448_d2i_public_key           NULL
 #  define ed448_d2i_key_params           NULL
 #  define ed448_d2i_PKCS8                ecx_d2i_PKCS8
-#  define ed448_d2i_PUBKEY               (d2i_of_void *)ossl_d2i_ED448_PUBKEY
 #  define ed448_free                     (free_key_fn *)ossl_ecx_key_free
 #  define ed448_check                    NULL
 #  define ed448_adjust                   ecx_key_adjust
@@ -508,7 +521,6 @@ static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx)
 #  define x25519_d2i_public_key          NULL
 #  define x25519_d2i_key_params          NULL
 #  define x25519_d2i_PKCS8               ecx_d2i_PKCS8
-#  define x25519_d2i_PUBKEY              (d2i_of_void *)ossl_d2i_X25519_PUBKEY
 #  define x25519_free                    (free_key_fn *)ossl_ecx_key_free
 #  define x25519_check                   NULL
 #  define x25519_adjust                  ecx_key_adjust
@@ -518,7 +530,6 @@ static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx)
 #  define x448_d2i_public_key            NULL
 #  define x448_d2i_key_params            NULL
 #  define x448_d2i_PKCS8                 ecx_d2i_PKCS8
-#  define x448_d2i_PUBKEY                (d2i_of_void *)ossl_d2i_X448_PUBKEY
 #  define x448_free                      (free_key_fn *)ossl_ecx_key_free
 #  define x448_check                     NULL
 #  define x448_adjust                    ecx_key_adjust
@@ -529,18 +540,17 @@ static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx)
 #  define sm2_d2i_private_key           (d2i_of_void *)d2i_ECPrivateKey
 #  define sm2_d2i_public_key            NULL
 #  define sm2_d2i_key_params            (d2i_of_void *)d2i_ECParameters
+#  define sm2_d2i_PUBKEY                ec_d2i_PUBKEY
+#  define sm2_free                      (free_key_fn *)EC_KEY_free
+#  define sm2_check                     ec_check
+#  define sm2_adjust                    ec_adjust
 
-static void *sm2_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
+static void *sm2_d2i_PKCS8(const unsigned char **der, long der_len,
                            struct der2key_ctx_st *ctx)
 {
     return der2key_decode_p8(der, der_len, ctx,
                              (key_from_pkcs8_t *)ossl_ec_key_from_pkcs8);
 }
-
-#  define sm2_d2i_PUBKEY                (d2i_of_void *)d2i_EC_PUBKEY
-#  define sm2_free                      (free_key_fn *)EC_KEY_free
-#  define sm2_check                     ec_check
-#  define sm2_adjust                    ec_adjust
 # endif
 #endif
 
@@ -550,16 +560,21 @@ static void *sm2_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
 #define rsa_d2i_private_key             (d2i_of_void *)d2i_RSAPrivateKey
 #define rsa_d2i_public_key              (d2i_of_void *)d2i_RSAPublicKey
 #define rsa_d2i_key_params              NULL
+#define rsa_free                        (free_key_fn *)RSA_free
 
-static void *rsa_d2i_PKCS8(void **key, const unsigned char **der, long der_len,
+static void *rsa_d2i_PKCS8(const unsigned char **der, long der_len,
                            struct der2key_ctx_st *ctx)
 {
     return der2key_decode_p8(der, der_len, ctx,
                              (key_from_pkcs8_t *)ossl_rsa_key_from_pkcs8);
 }
 
-#define rsa_d2i_PUBKEY                  (d2i_of_void *)d2i_RSA_PUBKEY
-#define rsa_free                        (free_key_fn *)RSA_free
+static void *
+rsa_d2i_PUBKEY(const unsigned char **der, long der_len,
+               ossl_unused struct der2key_ctx_st *ctx)
+{
+    return d2i_RSA_PUBKEY(NULL, der, der_len);
+}
 
 static int rsa_check(void *key, struct der2key_ctx_st *ctx)
 {
@@ -592,7 +607,7 @@ static void rsa_adjust(void *key, struct der2key_ctx_st *ctx)
 #define rsapss_d2i_public_key           (d2i_of_void *)d2i_RSAPublicKey
 #define rsapss_d2i_key_params           NULL
 #define rsapss_d2i_PKCS8                rsa_d2i_PKCS8
-#define rsapss_d2i_PUBKEY               (d2i_of_void *)d2i_RSA_PUBKEY
+#define rsapss_d2i_PUBKEY               rsa_d2i_PUBKEY
 #define rsapss_free                     (free_key_fn *)RSA_free
 #define rsapss_check                    rsa_check
 #define rsapss_adjust                   rsa_adjust
diff --git a/providers/implementations/encode_decode/encode_key2any.c b/providers/implementations/encode_decode/encode_key2any.c
index 2b39bf039f..e77cb22e8c 100644
--- a/providers/implementations/encode_decode/encode_key2any.c
+++ b/providers/implementations/encode_decode/encode_key2any.c
@@ -535,8 +535,6 @@ static int dh_check_key_type(const void *dh, int expected_type)
 
 # define dh_evp_type            EVP_PKEY_DH
 # define dhx_evp_type           EVP_PKEY_DHX
-# define dh_input_type          "DH"
-# define dhx_input_type         "DHX"
 # define dh_pem_type            "DH"
 # define dhx_pem_type           "X9.42 DH"
 #endif
@@ -632,7 +630,6 @@ static int dsa_pki_priv_to_der(const void *dsa, unsigned char **pder)
 
 # define dsa_check_key_type     NULL
 # define dsa_evp_type           EVP_PKEY_DSA
-# define dsa_input_type         "DSA"
 # define dsa_pem_type           "DSA"
 #endif
 
@@ -736,7 +733,6 @@ static int ec_pki_priv_to_der(const void *veckey, unsigned char **pder)
 
 # define ec_check_key_type      NULL
 # define ec_evp_type            EVP_PKEY_EC
-# define ec_input_type          "EC"
 # define ec_pem_type            "EC"
 
 # ifndef OPENSSL_NO_SM2
@@ -749,7 +745,6 @@ static int ec_pki_priv_to_der(const void *veckey, unsigned char **pder)
  * in AlgorithmIdentified.params).
  */
 #  define sm2_evp_type          ec_evp_type
-#  define sm2_input_type        "SM2"
 #  define sm2_pem_type          "SM2"
 # endif
 #endif
@@ -814,10 +809,6 @@ static int ecx_pki_priv_to_der(const void *vecxkey, unsigned char **pder)
 # define ed448_evp_type         EVP_PKEY_ED448
 # define x25519_evp_type        EVP_PKEY_X25519
 # define x448_evp_type          EVP_PKEY_X448
-# define ed25519_input_type     "ED25519"
-# define ed448_input_type       "ED448"
-# define x25519_input_type      "X25519"
-# define x448_input_type        "X448"
 # define ed25519_pem_type       "ED25519"
 # define ed448_pem_type         "ED448"
 # define x25519_pem_type        "X25519"
@@ -930,8 +921,6 @@ static int rsa_check_key_type(const void *rsa, int expected_type)
 
 #define rsa_evp_type            EVP_PKEY_RSA
 #define rsapss_evp_type         EVP_PKEY_RSA_PSS
-#define rsa_input_type          "RSA"
-#define rsapss_input_type       "RSA-PSS"
 #define rsa_pem_type            "RSA"
 #define rsapss_pem_type         "RSA-PSS"
 
diff --git a/providers/implementations/encode_decode/encode_key2text.c b/providers/implementations/encode_decode/encode_key2text.c
index db0c8abe82..7a47c61ee8 100644
--- a/providers/implementations/encode_decode/encode_key2text.c
+++ b/providers/implementations/encode_decode/encode_key2text.c
@@ -282,9 +282,6 @@ static int dh_to_text(BIO *out, const void *key, int selection)
 
     return 1;
 }
-
-# define dh_input_type          "DH"
-# define dhx_input_type         "DHX"
 #endif
 
 /* ---------------------------------------------------------------------- */
@@ -352,8 +349,6 @@ static int dsa_to_text(BIO *out, const void *key, int selection)
 
     return 1;
 }
-
-# define dsa_input_type         "DSA"
 #endif
 
 /* ---------------------------------------------------------------------- */
@@ -557,12 +552,6 @@ err:
     OPENSSL_free(pub);
     return ret;
 }
-
-# define ec_input_type          "EC"
-
-# ifndef OPENSSL_NO_SM2
-#  define sm2_input_type        "SM2"
-# endif
 #endif
 
 /* ---------------------------------------------------------------------- */
@@ -620,10 +609,6 @@ static int ecx_to_text(BIO *out, const void *key, int selection)
     return 1;
 }
 
-# define ed25519_input_type     "ED25519"
-# define ed448_input_type       "ED448"
-# define x25519_input_type      "X25519"
-# define x448_input_type        "X448"
 #endif
 
 /* ---------------------------------------------------------------------- */
@@ -780,9 +765,6 @@ static int rsa_to_text(BIO *out, const void *key, int selection)
     return ret;
 }
 
-#define rsa_input_type          "RSA"
-#define rsapss_input_type       "RSA-PSS"
-
 /* ---------------------------------------------------------------------- */
 
 static void *key2text_newctx(void *provctx)