Fix a possible memleak in rsa_pub_encode

That seems to be only an issue for RSA-PSS with parameters.
Spotted by code review, so it looks like there is no test coverage for this.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22032)

crypto/rsa/rsa_ameth.c

@@ -60,13 +60,16 @@ static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
     if (!rsa_param_encode(pkey, &str, &strtype))
         return 0;
     penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
-    if (penclen <= 0)
+    if (penclen <= 0) {
+        ASN1_STRING_free(str);
         return 0;
+    }
     if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id),
                                strtype, str, penc, penclen))
         return 1;
 
     OPENSSL_free(penc);
+    ASN1_STRING_free(str);
     return 0;
 }
 

test/recipes/15-test_rsapss.t

@@ -16,7 +16,7 @@ use OpenSSL::Test::Utils;
 
 setup("test_rsapss");
 
-plan tests => 16;
+plan tests => 18;
 
 #using test/testrsa.pem which happens to be a 512 bit RSA
 ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha1',
@@ -124,3 +124,11 @@ ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
 ok(!run(app([ 'openssl', 'rsa',
              '-in' => data_file('negativesaltlen.pem')],
              '-out' => 'badout')));
+
+ok(run(app(['openssl', 'genpkey', '-algorithm', 'RSA-PSS', '-pkeyopt', 'rsa_keygen_bits:1024',
+            '-pkeyopt', 'rsa_pss_keygen_md:SHA256', '-pkeyopt', 'rsa_pss_keygen_saltlen:10',
+            '-out', 'testrsapss.pem'])),
+   "openssl genpkey RSA-PSS with pss parameters");
+ok(run(app(['openssl', 'pkey', '-in', 'testrsapss.pem', '-pubout', '-text'])),
+   "openssl pkey, execute rsa_pub_encode with pss parameters");
+unlink 'testrsapss.pem';