Fix Coverity issues in HPKE

CID 1517043 and 1517038: (Forward NULL) - Removed redundant check that is already
done by the caller. It was complaining that it checked for ctlen == NULL
and then did a goto that used this *ctlen.

CID 1517042 and 1517041: (Forward NULL) - Similar to above for ptlen in
hpke_aead_dec()

CID 1517040: Remove unneeded logging. This gets rid of the warning
related to taking the sizeof(&)

CID 1517039: Check returned value of  RAND_bytes_ex() in hpke_test

CID 1517038: Check return result of KEM_INFO_find() in
OSSL_HPKE_get_recomended_ikmelen. Even though this is a false positive,
it should not rely on the internals of other function calls.

Changed some goto's into returns to match OpenSSL coding guidelines.
Removed Raises from calls to _new which fail from malloc calls.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19774)

crypto/hpke/hpke.c

@@ -155,25 +155,20 @@ static int hpke_aead_dec(OSSL_LIB_CTX *libctx, const char *propq,
     EVP_CIPHER *enc = NULL;
     const OSSL_HPKE_AEAD_INFO *aead_info = NULL;
 
-    if (pt == NULL || ptlen == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
-        goto err;
-    }
     aead_info = ossl_HPKE_AEAD_INFO_find_id(suite.aead_id);
     if (aead_info == NULL) {
         ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
-        goto err;
+        return 0;
     }
     taglen = aead_info->taglen;
     if (ctlen <= taglen || *ptlen < ctlen - taglen) {
         ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT);
-        goto err;
+        return 0;
     }
     /* Create and initialise the context */
-    if ((ctx = EVP_CIPHER_CTX_new()) == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
-        goto err;
-    }
+    if ((ctx = EVP_CIPHER_CTX_new()) == NULL)
+        return 0;
+
     /* Initialise the encryption operation */
     enc = EVP_CIPHER_fetch(libctx, aead_info->name, propq);
     if (enc == NULL) {
@@ -260,25 +255,20 @@ static int hpke_aead_enc(OSSL_LIB_CTX *libctx, const char *propq,
     EVP_CIPHER *enc = NULL;
     unsigned char tag[16];
 
-    if (ct == NULL || ctlen == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
-        goto err;
-    }
     aead_info = ossl_HPKE_AEAD_INFO_find_id(suite.aead_id);
     if (aead_info == NULL) {
         ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
-        goto err;
+        return 0;
     }
     taglen = aead_info->taglen;
     if (*ctlen <= taglen || ptlen > *ctlen - taglen) {
         ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT);
-        goto err;
+        return 0;
     }
     /* Create and initialise the context */
-    if ((ctx = EVP_CIPHER_CTX_new()) == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
-        goto err;
-    }
+    if ((ctx = EVP_CIPHER_CTX_new()) == NULL)
+        return 0;
+
     /* Initialise the encryption operation. */
     enc = EVP_CIPHER_fetch(libctx, aead_info->name, propq);
     if (enc == NULL) {
@@ -1435,5 +1425,8 @@ size_t OSSL_HPKE_get_recommended_ikmelen(OSSL_HPKE_SUITE suite)
     if (hpke_suite_check(suite) != 1)
         return 0;
     kem_info = ossl_HPKE_KEM_INFO_find_id(suite.kem_id);
+    if (kem_info == NULL)
+        return 0;
+
     return kem_info->Nsk;
 }

test/hpke_test.c

@@ -1008,11 +1008,10 @@ static int test_hpke_modes_suites(void)
                             overallresult = 0;
                     }
                     if (COIN_IS_HEADS) {
-                        RAND_bytes_ex(testctx,
-                                      (unsigned char *) &startseq,
-                                      sizeof(startseq),
-                                      RAND_DRBG_STRENGTH);
-                        if (!TEST_true(OSSL_HPKE_CTX_set_seq(ctx, startseq)))
+                        if (!TEST_int_eq(1, RAND_bytes_ex(testctx,
+                                                          (unsigned char *) &startseq,
+                                                          sizeof(startseq), 0))
+                            || !TEST_true(OSSL_HPKE_CTX_set_seq(ctx, startseq)))
                             overallresult = 0;
                     } else {
                         startseq = 0;
@@ -1207,8 +1206,6 @@ static int test_hpke_suite_strs(void)
     char sstr[128];
     OSSL_HPKE_SUITE stirred;
     char giant[2048];
-    size_t suitesize;
-    size_t ptr_suitesize;
 
     for (kemind = 0; kemind != OSSL_NELEM(kem_str_list); kemind++) {
         for (kdfind = 0; kdfind != OSSL_NELEM(kdf_str_list); kdfind++) {
@@ -1245,14 +1242,6 @@ static int test_hpke_suite_strs(void)
     if (!TEST_false(OSSL_HPKE_str2suite(giant, &stirred)))
         overallresult = 0;
 
-    /* we'll check the size of a suite just to see what we get */
-    suitesize = sizeof(stirred);
-    ptr_suitesize = sizeof(&stirred);
-    if (verbose) {
-        TEST_note("Size of OSSL_HPKE_SUITE is %d, size of ptr is %d",
-                  (int) suitesize, (int) ptr_suitesize);
-    }
-
     return overallresult;
 }