apps/cms.c: Make -sign and -verify handle binary input

Fixes #8940

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12959)

3 files changed, 86 insertions, 16 deletions

diff --git a/apps/cms.c b/apps/cms.c
index f40049edac..64867e3702 100644
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -159,7 +159,7 @@ const OPTIONS cms_options[] = {
     {"nodetach", OPT_NODETACH, '-', "Use opaque signing"},
     {"nosmimecap", OPT_NOSMIMECAP, '-', "Omit the SMIMECapabilities attribute"},
     {"noattr", OPT_NOATTR, '-', "Don't include any signed attributes"},
-    {"binary", OPT_BINARY, '-', "Don't translate message to text"},
+    {"binary", OPT_BINARY, '-', "Treat input as binary: do not translate to canonical form"},
     {"keyid", OPT_KEYID, '-', "Use subject key identifier"},
     {"nosigs", OPT_NOSIGS, '-', "Don't verify message signature"},
     {"nocerts", OPT_NOCERTS, '-',
@@ -227,7 +227,7 @@ const OPTIONS cms_options[] = {
     {NULL}
 };
 
-static CMS_ContentInfo *load_content_info(int informat, BIO *in, BIO **indata,
+static CMS_ContentInfo *load_content_info(int informat, BIO *in, int flags, BIO **indata,
                                           const char *name,
                                           OSSL_LIB_CTX *libctx,
                                           const char *propq)
@@ -241,7 +241,7 @@ static CMS_ContentInfo *load_content_info(int informat, BIO *in, BIO **indata,
     }
     switch (informat) {
     case FORMAT_SMIME:
-        ci = SMIME_read_CMS_ex(in, indata, &ret);
+        ci = SMIME_read_CMS_ex(in, flags, indata, &ret);
         break;
     case FORMAT_PEM:
         ci = PEM_read_bio_CMS(in, &ret, NULL, NULL);
@@ -263,6 +263,29 @@ err:
     return NULL;
 }
 
+static void warn_binary(const char *file)
+{
+    BIO *bio;
+    unsigned char linebuf[1024], *cur, *end;
+    int len;
+
+    if ((bio = bio_open_default(file, 'r', FORMAT_BINARY)) == NULL)
+        return; /* cannot give a proper warning since there is an error */
+    while ((len = BIO_read(bio, linebuf, sizeof(linebuf))) > 0) {
+        end = linebuf + len;
+        for (cur = linebuf; cur < end; cur++) {
+            if (*cur == '\0' || *cur >= 0x80) {
+                BIO_printf(bio_err, "Warning: input file '%s' contains %s"
+                           " character; better use -binary option\n",
+                           file, *cur == '\0' ? "NUL" : "8-bit");
+                break;
+            }
+        }
+    }
+    BIO_free(bio);
+}
+
+
 int cms_main(int argc, char **argv)
 {
     CONF *conf = NULL;
@@ -452,7 +475,7 @@ int cms_main(int argc, char **argv)
                                 OPT_FMT_PEMDER | OPT_FMT_SMIME, &rctformat))
                     goto opthelp;
             } else {
-                rcms = load_content_info(rctformat, rctin, NULL, "recipient",
+                rcms = load_content_info(rctformat, rctin, 0, NULL, "recipient",
                                          libctx, app_get0_propq());
             }
             break;
@@ -784,13 +807,12 @@ int cms_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~CMS_DETACHED;
 
-    if (!(operation & SMIME_OP))
-        if (flags & CMS_BINARY)
+    if ((flags & CMS_BINARY) != 0) {
+        if (!(operation & SMIME_OP))
             outformat = FORMAT_BINARY;
-
-    if (!(operation & SMIME_IP))
-        if (flags & CMS_BINARY)
+        if (!(operation & SMIME_IP))
             informat = FORMAT_BINARY;
+    }
 
     if (operation == SMIME_ENCRYPT) {
         if (!cipher) {
@@ -867,16 +889,22 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, 'r', informat);
+    if ((flags & CMS_BINARY) == 0)
+        warn_binary(infile);
+    in = bio_open_default(infile, 'r',
+                          (flags & CMS_BINARY) != 0 ? FORMAT_BINARY : informat);
     if (in == NULL)
         goto end;
 
     if (operation & SMIME_IP) {
-        cms = load_content_info(informat, in, &indata, "SMIME", libctx, app_get0_propq());
+        cms = load_content_info(informat, in, flags, &indata, "SMIME",
+                                libctx, app_get0_propq());
         if (cms == NULL)
             goto end;
         if (contfile != NULL) {
             BIO_free(indata);
+            if ((flags & CMS_BINARY) == 0)
+                warn_binary(contfile);
             if ((indata = BIO_new_file(contfile, "rb")) == NULL) {
                 BIO_printf(bio_err, "Can't read content file %s\n", contfile);
                 goto end;
@@ -897,13 +925,14 @@ int cms_main(int argc, char **argv)
 
     if (rctfile != NULL) {
         char *rctmode = (rctformat == FORMAT_ASN1) ? "rb" : "r";
+
         if ((rctin = BIO_new_file(rctfile, rctmode)) == NULL) {
             BIO_printf(bio_err, "Can't open receipt file %s\n", rctfile);
             goto end;
         }
 
-        rcms = load_content_info(rctformat, rctin, NULL, "recipient", libctx,
-                                 app_get0_propq());
+        rcms = load_content_info(rctformat, rctin, 0, NULL, "recipient", libctx,
+                                 app_get0_propq);
         if (rcms == NULL)
             goto end;
     }
diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index a371f21ad8..0e20b807c8 100644
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -12,7 +12,7 @@ use warnings;
 
 use POSIX;
 use File::Spec::Functions qw/catfile/;
-use File::Compare qw/compare_text/;
+use File::Compare qw/compare_text compare/;
 use OpenSSL::Test qw/:DEFAULT srctop_dir srctop_file bldtop_dir bldtop_file/;
 
 use OpenSSL::Test::Utils;
@@ -50,8 +50,7 @@ my ($no_des, $no_dh, $no_dsa, $no_ec, $no_ec2m, $no_rc2, $no_zlib)
 
 $no_rc2 = 1 if disabled("legacy");
 
-plan tests =>
-    + 10;
+plan tests => 11;
 
 unless ($no_fips) {
     @config = ( "-config", srctop_file("test", "fips-and-base.cnf") );
@@ -812,6 +811,48 @@ subtest "CAdES ko tests\n" => sub {
     }
 };
 
+subtest "CMS binary input tests\n" => sub {
+    my $input = srctop_file("test", "smcont.bin");
+    my $signed = "smcont.signed";
+    my $verified = "smcont.verified";
+    my $cert = srctop_file("test", "certs", "ee-self-signed.pem");
+    my $key = srctop_file("test", "certs", "ee-key.pem");
+
+    plan tests => 11;
+
+    ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
+                "-signer", $cert, "-inkey", $key,
+                "-binary", "-in", $input, "-out", $signed])),
+       "sign binary input with -binary");
+    ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
+                "-binary", "-in", $signed, "-out", $verified])),
+       "verify binary input with -binary");
+    is(compare($input, $verified), 0, "binary input retained with -binary");
+    ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
+                "-signer", $cert, "-inkey", $key,
+                "-in", $input, "-out", $signed])),
+       "sign binary input without -binary");
+    ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
+                "-in", $signed, "-out", $verified])),
+       "verify binary input without -binary");
+    is(compare($input, $verified), 1, "binary input not retained without -binary");
+    ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
+                "-binary", "-in", $signed, "-out", $verified])),
+       "verify binary input wrong crlfeol");
+
+    ok(run(app(["openssl", "cms", "-sign", "-md", "sha256", "-crlfeol",
+                "-signer", $cert, "-inkey", $key,
+                "-binary", "-in", $input, "-out", $signed.".crlf"])),
+       "sign binary input crlfeol");
+    ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
+                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
+       "verify binary input crlfeol");
+    is(compare($input, $verified.".crlf"), 0);
+    ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
+                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
+       "verify binary input missing crlfeol");
+};
+
 sub check_availability {
     my $tnam = shift;
 
diff --git a/test/smcont.bin b/test/smcont.bin
new file mode 100644
index 0000000000..2a5ce10224
--- /dev/null
+++ b/test/smcont.bin