EVP: Enforce that EVP_PKEY_set_alias_type() only works with legacy keys

This also deprecates the function, as it is not necessary any more,
and should fall out of use.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12920)

CHANGES.md

@@ -23,6 +23,15 @@ OpenSSL 3.0
 
 ### Changes between 1.1.1 and 3.0 [xx XXX xxxx]
 
+ * Deprecated EVP_PKEY_set_alias_type().  This function was previously
+   needed as a workaround to recognise SM2 keys.  With OpenSSL 3.0, this key
+   type is internally recognised so the workaround is no longer needed.
+
+   Functionality is still retained as it is, but will only work with
+   EVP_PKEYs with a legacy internal key.
+
+   *Richard Levitte*
+
  * Changed all "STACK" functions to be macros instead of inline functions. Macro
    parameters are still checked for type safety at compile time via helper
    inline functions.

crypto/evp/p_lib.c

@@ -661,8 +661,19 @@ int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len)
     return pkey_set_type(pkey, NULL, EVP_PKEY_NONE, str, len, NULL);
 }
 
+#ifndef OPENSSL_NO_DEPRECATED_3_0
 int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type)
 {
+    if (!evp_pkey_is_legacy(pkey)) {
+        const char *name = OBJ_nid2sn(type);
+
+        if (name != NULL && EVP_PKEY_is_a(pkey, name))
+            return 1;
+
+        ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION);
+        return 0;
+    }
+
     if (pkey->type == type) {
         return 1; /* it already is that type */
     }
@@ -679,6 +690,7 @@ int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type)
     pkey->type = type;
     return 1;
 }
+#endif
 
 # ifndef OPENSSL_NO_ENGINE
 int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *e)

doc/man3/EVP_PKEY_set1_RSA.pod

@@ -43,11 +43,16 @@ EVP_PKEY_set1_engine, EVP_PKEY_get0_engine - EVP_PKEY assignment functions
  int EVP_PKEY_id(const EVP_PKEY *pkey);
  int EVP_PKEY_base_id(const EVP_PKEY *pkey);
  int EVP_PKEY_type(int type);
- int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type);
 
  ENGINE *EVP_PKEY_get0_engine(const EVP_PKEY *pkey);
  int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *engine);
 
+Deprecated since OpenSSL 3.0, can be hidden entirely by defining
+B<OPENSSL_API_COMPAT> with a suitable version value, see
+L<openssl_user_macros(7)>:
+
+ int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type);
+
 =head1 DESCRIPTION
 
 EVP_PKEY_set1_RSA(), EVP_PKEY_set1_DSA(), EVP_PKEY_set1_DH() and
@@ -163,6 +168,10 @@ algorithms with EVP_PKEY_set_alias_type:
 
 L<EVP_PKEY_new(3)>, L<SM2(7)>
 
+=head1 HISTORY
+
+EVP_PKEY_set_alias_type() was deprecated in OpenSSL 3.0.
+
 =head1 COPYRIGHT
 
 Copyright 2002-2020 The OpenSSL Project Authors. All Rights Reserved.

include/openssl/evp.h

@@ -1195,7 +1195,7 @@ int EVP_PKEY_can_sign(const EVP_PKEY *pkey);
 int EVP_PKEY_set_type(EVP_PKEY *pkey, int type);
 int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len);
 int EVP_PKEY_set_type_by_keymgmt(EVP_PKEY *pkey, EVP_KEYMGMT *keymgmt);
-int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type);
+DEPRECATEDIN_3_0(int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type))
 # ifndef OPENSSL_NO_ENGINE
 int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *e);
 ENGINE *EVP_PKEY_get0_engine(const EVP_PKEY *pkey);

util/libcrypto.num

@@ -4394,7 +4394,7 @@ EVP_PKEY_get_raw_public_key             4518	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_get_raw_private_key            4519	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_asn1_set_get_priv_key          4520	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_asn1_set_get_pub_key           4521	3_0_0	EXIST::FUNCTION:
-EVP_PKEY_set_alias_type                 4522	3_0_0	EXIST::FUNCTION:
+EVP_PKEY_set_alias_type                 4522	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3_0
 RAND_keep_random_devices_open           4523	3_0_0	EXIST::FUNCTION:
 EC_POINT_set_compressed_coordinates     4524	3_0_0	EXIST::FUNCTION:EC
 EC_POINT_set_affine_coordinates         4525	3_0_0	EXIST::FUNCTION:EC