Limit size of modulus for bn_mul_mont and BN_mod_exp_mont_consttime

Otherwise the alloca can cause an exception.

Issue reported by Jiayi Lin.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/20005)

1 files changed, 29 insertions, 0 deletions

diff --git a/test/exptest.c b/test/exptest.c
index 8baf8c37ac..ac2e6d3756 100644
--- a/test/exptest.c
+++ b/test/exptest.c
@@ -50,6 +50,7 @@ static int test_mod_exp_zero(void)
     BN_ULONG one_word = 1;
     BN_CTX *ctx = BN_CTX_new();
     int ret = 0, failed = 0;
+    BN_MONT_CTX *mont = NULL;
 
     if (!TEST_ptr(m = BN_new())
         || !TEST_ptr(a = BN_new())
@@ -94,6 +95,33 @@ static int test_mod_exp_zero(void)
     if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a)))
         failed = 1;
 
+    if (!TEST_ptr(mont = BN_MONT_CTX_new()))
+        goto err;
+
+    ERR_set_mark();
+    /* mont is not set but passed in */
+    if (!TEST_false(BN_mod_exp_mont_consttime(r, p, a, m, ctx, mont)))
+        goto err;
+    if (!TEST_false(BN_mod_exp_mont(r, p, a, m, ctx, mont)))
+        goto err;
+    ERR_pop_to_mark();
+
+    if (!TEST_true(BN_MONT_CTX_set(mont, m, ctx)))
+        goto err;
+
+    /* we compute 0 ** a mod 1 here, to execute code that uses mont */
+    if (!TEST_true(BN_mod_exp_mont_consttime(r, p, a, m, ctx, mont)))
+        goto err;
+
+    if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a)))
+        failed = 1;
+
+    if (!TEST_true(BN_mod_exp_mont(r, p, a, m, ctx, mont)))
+        goto err;
+
+    if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont", r, a)))
+        failed = 1;
+
     /*
      * A different codepath exists for single word multiplication
      * in non-constant-time only.
@@ -114,6 +142,7 @@ static int test_mod_exp_zero(void)
     BN_free(a);
     BN_free(p);
     BN_free(m);
+    BN_MONT_CTX_free(mont);
     BN_CTX_free(ctx);
 
     return ret;