apps/dgst.c: allocate a new signature buffer

... if the fixed-size buffer is too small.

Fixes #9732

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Patrick Steuer <patrick.steuer@de.ibm.com>
(Merged from https://github.com/openssl/openssl/pull/10276)

apps/dgst.c

@@ -502,15 +502,16 @@ int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
           const char *sig_name, const char *md_name,
           const char *file)
 {
-    size_t len;
-    int i, backslash = 0;
+    size_t len = BUFSIZE;
+    int i, backslash = 0, ret = 1;
+    unsigned char *sigbuf = NULL;
 
     while (BIO_pending(bp) || !BIO_eof(bp)) {
         i = BIO_read(bp, (char *)buf, BUFSIZE);
         if (i < 0) {
             BIO_printf(bio_err, "Read Error in %s\n", file);
             ERR_print_errors(bio_err);
-            return 1;
+            goto end;
         }
         if (i == 0)
             break;
@@ -523,28 +524,35 @@ int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
             BIO_printf(out, "Verified OK\n");
         } else if (i == 0) {
             BIO_printf(out, "Verification Failure\n");
-            return 1;
+            goto end;
         } else {
             BIO_printf(bio_err, "Error Verifying Data\n");
             ERR_print_errors(bio_err);
-            return 1;
+            goto end;
         }
-        return 0;
+        ret = 0;
+        goto end;
     }
     if (key != NULL) {
         EVP_MD_CTX *ctx;
+        int pkey_len;
         BIO_get_md_ctx(bp, &ctx);
-        len = BUFSIZE;
+        pkey_len = EVP_PKEY_size(key);
+        if (pkey_len > BUFSIZE) {
+            len = pkey_len;
+            sigbuf = app_malloc(len, "Signature buffer");
+            buf = sigbuf;
+        }
         if (!EVP_DigestSignFinal(ctx, buf, &len)) {
             BIO_printf(bio_err, "Error Signing Data\n");
             ERR_print_errors(bio_err);
-            return 1;
+            goto end;
         }
     } else {
         len = BIO_gets(bp, (char *)buf, BUFSIZE);
         if ((int)len < 0) {
             ERR_print_errors(bio_err);
-            return 1;
+            goto end;
         }
     }
 
@@ -579,5 +587,11 @@ int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
         }
         BIO_printf(out, "\n");
     }
-    return 0;
+
+    ret = 0;
+ end:
+    if (sigbuf != NULL)
+        OPENSSL_clear_free(sigbuf, len);
+
+    return ret;
 }