Remove the function EVP_PKEY_set_alias_type

OTC recently voted that EVP_PKEY types will be immutable in 3.0. This
means that EVP_PKEY_set_alias_type can no longer work and should be
removed entirely (applications will need to be rewritten not to use it).

It was primarily used for SM2 which no longer needs this call.
Applications should generate SM2 keys directly (without going via an EC
key first), or otherwise when loading keys they should automatically be
detected as SM2 keys.

Fixes #14379

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14803)

CHANGES.md

@@ -432,12 +432,11 @@ OpenSSL 3.0
 
    *Paul Dale*
 
- * Deprecated EVP_PKEY_set_alias_type().  This function was previously
+ * Removed EVP_PKEY_set_alias_type().  This function was previously
    needed as a workaround to recognise SM2 keys.  With OpenSSL 3.0, this key
    type is internally recognised so the workaround is no longer needed.
 
-   Functionality is still retained as it is, but will only work with
-   EVP_PKEYs with a legacy internal key.
+   This is a breaking change from previous OpenSSL versions.
 
    *Richard Levitte*
 
@@ -863,16 +862,16 @@ OpenSSL 3.0
    *Paul Dale*
 
  * Reworked the treatment of EC EVP_PKEYs with the SM2 curve to
-   automatically become EVP_PKEY_SM2 rather than EVP_PKEY_EC.
-   This means that applications don't have to look at the curve NID and
-   `EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)` to get SM2 computations.
-   However, they still can, that `EVP_PKEY_set_alias_type()` call acts as
-   a no-op when the EVP_PKEY is already of the given type.
+   automatically become EVP_PKEY_SM2 rather than EVP_PKEY_EC. This is a breaking
+   change from previous OpenSSL versions.
+
+   Unlike in previous OpenSSL versions, this means that applications must not
+   call `EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)` to get SM2 computations.
+   The `EVP_PKEY_set_alias_type` function has now been removed.
 
    Parameter and key generation is also reworked to make it possible
-   to generate EVP_PKEY_SM2 parameters and keys without having to go
-   through EVP_PKEY_EC generation and then change the EVP_PKEY type.
-   However, code that does the latter will still work as before.
+   to generate EVP_PKEY_SM2 parameters and keys. Applications must now generate
+   SM2 keys directly and must not create an EVP_PKEY_EC key first.
 
    *Richard Levitte*
 

crypto/evp/p_lib.c

@@ -662,37 +662,6 @@ int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len)
     return pkey_set_type(pkey, NULL, EVP_PKEY_NONE, str, len, NULL);
 }
 
-# ifndef OPENSSL_NO_DEPRECATED_3_0
-int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type)
-{
-    if (!evp_pkey_is_legacy(pkey)) {
-        const char *name = OBJ_nid2sn(type);
-
-        if (name != NULL && EVP_PKEY_is_a(pkey, name))
-            return 1;
-
-        ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION);
-        return 0;
-    }
-
-    if (pkey->type == type) {
-        return 1; /* it already is that type */
-    }
-
-    /*
-     * The application is requesting to alias this to a different pkey type,
-     * but not one that resolves to the base type.
-     */
-    if (EVP_PKEY_type(type) != EVP_PKEY_base_id(pkey)) {
-        ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_ALGORITHM);
-        return 0;
-    }
-
-    pkey->type = type;
-    return 1;
-}
-# endif
-
 # ifndef OPENSSL_NO_ENGINE
 int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *e)
 {
@@ -721,21 +690,31 @@ ENGINE *EVP_PKEY_get0_engine(const EVP_PKEY *pkey)
 # ifndef OPENSSL_NO_DEPRECATED_3_0
 int EVP_PKEY_assign(EVP_PKEY *pkey, int type, void *key)
 {
-    int alias = type;
-
 #  ifndef OPENSSL_NO_EC
-    if ((key != NULL) && (EVP_PKEY_type(type) == EVP_PKEY_EC)) {
+    int pktype;
+
+    pktype = EVP_PKEY_type(type);
+    if ((key != NULL) && (pktype == EVP_PKEY_EC || pktype == EVP_PKEY_SM2)) {
         const EC_GROUP *group = EC_KEY_get0_group(key);
 
-        if (group != NULL && EC_GROUP_get_curve_name(group) == NID_sm2)
-            alias = EVP_PKEY_SM2;
+        if (group != NULL) {
+            int curve = EC_GROUP_get_curve_name(group);
+
+            /*
+             * Regardless of what is requested the SM2 curve must be SM2 type,
+             * and non SM2 curves are EC type.
+             */
+            if (curve == NID_sm2 && pktype == EVP_PKEY_EC)
+                type = EVP_PKEY_SM2;
+            else if(curve != NID_sm2 && pktype == EVP_PKEY_SM2)
+                type = EVP_PKEY_EC;
+        }
     }
 #  endif
 
     if (pkey == NULL || !EVP_PKEY_set_type(pkey, type))
         return 0;
-    if (!EVP_PKEY_set_alias_type(pkey, alias))
-        return 0;
+
     pkey->pkey.ptr = key;
     return (key != NULL);
 }
@@ -1505,10 +1484,12 @@ static int pkey_set_type(EVP_PKEY *pkey, ENGINE *e, int type, const char *str,
          * EVP_PKEY_KEYMGMT, which indicates that one should be cautious
          * with functions that expect legacy internal keys.
          */
-        if (ameth != NULL)
-            pkey->type = ameth->pkey_id;
-        else
+        if (ameth != NULL) {
+            if (type == EVP_PKEY_NONE)
+                pkey->type = ameth->pkey_id;
+        } else {
             pkey->type = EVP_PKEY_KEYMGMT;
+        }
 #endif
     }
     return 1;

doc/man3/EVP_PKEY_set1_RSA.pod

@@ -9,7 +9,7 @@ EVP_PKEY_assign_RSA, EVP_PKEY_assign_DSA, EVP_PKEY_assign_DH,
 EVP_PKEY_assign_EC_KEY, EVP_PKEY_assign_POLY1305, EVP_PKEY_assign_SIPHASH,
 EVP_PKEY_get0_hmac, EVP_PKEY_get0_poly1305, EVP_PKEY_get0_siphash,
 EVP_PKEY_get0, EVP_PKEY_type, EVP_PKEY_id, EVP_PKEY_base_id,
-EVP_PKEY_set_alias_type, EVP_PKEY_set1_engine, EVP_PKEY_get0_engine -
+EVP_PKEY_set1_engine, EVP_PKEY_get0_engine -
 EVP_PKEY assignment functions
 
 =head1 SYNOPSIS
@@ -24,8 +24,6 @@ Deprecated since OpenSSL 3.0, can be hidden entirely by defining
 B<OPENSSL_API_COMPAT> with a suitable version value, see
 L<openssl_user_macros(7)>:
 
- int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type);
-
  int EVP_PKEY_set1_RSA(EVP_PKEY *pkey, RSA *key);
  int EVP_PKEY_set1_DSA(EVP_PKEY *pkey, DSA *key);
  int EVP_PKEY_set1_DH(EVP_PKEY *pkey, DH *key);
@@ -130,19 +128,12 @@ If I<engine> does not include an B<EVP_PKEY_METHOD> for I<pkey> an
 error occurs. This function is deprecated. Applications should use providers
 instead of engines (see L<provider(7)> for details).
 
-EVP_PKEY_set_alias_type() allows modifying an EVP_PKEY to use a
-different set of algorithms than the default. This function is deprecated and
-was previously needed as a workaround to recognise SM2 keys. From OpenSSL 3.0,
-this key type is internally recognised so the workaround is no longer needed.
-Functionality is still retained as it is, but will only work with EVP_PKEYs
-with a legacy internal key.
-
 =head1 WARNINGS
 
 The following functions are only reliable with B<EVP_PKEY>s that have
 been assigned an internal key with EVP_PKEY_assign_*():
 
-EVP_PKEY_id(), EVP_PKEY_base_id(), EVP_PKEY_type(), EVP_PKEY_set_alias_type()
+EVP_PKEY_id(), EVP_PKEY_base_id(), EVP_PKEY_type()
 
 For EVP_PKEY key type checking purposes, L<EVP_PKEY_is_a(3)> is more generic.
 
@@ -171,12 +162,6 @@ EVP_PKEY_assign_EC_KEY() looks at the curve name id to determine if
 the passed B<EC_KEY> is an L<SM2(7)> key, and will set the B<EVP_PKEY>
 type to B<EVP_PKEY_SM2> in that case, instead of B<EVP_PKEY_EC>.
 
-It's possible to switch back and forth between the types B<EVP_PKEY_EC>
-and B<EVP_PKEY_SM2> with a call to EVP_PKEY_set_alias_type() on keys
-assigned with this macro if it's desirable to do a normal EC
-computations with the SM2 curve instead of the special SM2
-computations, and vice versa.
-
 Most applications wishing to know a key type will simply call
 EVP_PKEY_base_id() and will not care about the actual type:
 which will be identical in almost all cases.
@@ -206,15 +191,6 @@ type or B<NID_undef> (equivalently B<EVP_PKEY_NONE>) on error.
 
 EVP_PKEY_set1_engine() returns 1 for success and 0 for failure.
 
-EVP_PKEY_set_alias_type() returns 1 for success and 0 for error.
-
-=head1 EXAMPLES
-
-After loading an ECC key, it is possible to convert it to using SM2
-algorithms with EVP_PKEY_set_alias_type:
-
- EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2);
-
 =head1 SEE ALSO
 
 L<EVP_PKEY_new(3)>, L<SM2(7)>
@@ -227,12 +203,14 @@ EVP_PKEY_get0_RSA, EVP_PKEY_get0_DSA, EVP_PKEY_get0_DH, EVP_PKEY_get0_EC_KEY,
 EVP_PKEY_assign_RSA, EVP_PKEY_assign_DSA, EVP_PKEY_assign_DH,
 EVP_PKEY_assign_EC_KEY, EVP_PKEY_assign_POLY1305, EVP_PKEY_assign_SIPHASH,
 EVP_PKEY_get0_hmac, EVP_PKEY_get0_poly1305, EVP_PKEY_get0_siphash,
-EVP_PKEY_set_alias_type, EVP_PKEY_set1_engine and EVP_PKEY_get0_engine were
-deprecated in OpenSSL 3.0.
+EVP_PKEY_set1_engine and EVP_PKEY_get0_engine were deprecated in OpenSSL 3.0.
 
 The return value from EVP_PKEY_get0_RSA, EVP_PKEY_get0_DSA, EVP_PKEY_get0_DH,
 EVP_PKEY_get0_EC_KEY were made const in OpenSSL 3.0.
 
+The function EVP_PKEY_set_alias_type() was previously documented on this page.
+It was removed in OpenSSL 3.0.
+
 =head1 COPYRIGHT
 
 Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved.

include/openssl/evp.h

@@ -1262,7 +1262,6 @@ int EVP_PKEY_set_type(EVP_PKEY *pkey, int type);
 int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len);
 int EVP_PKEY_set_type_by_keymgmt(EVP_PKEY *pkey, EVP_KEYMGMT *keymgmt);
 # ifndef OPENSSL_NO_DEPRECATED_3_0
-OSSL_DEPRECATEDIN_3_0 int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type);
 #  ifndef OPENSSL_NO_ENGINE
 OSSL_DEPRECATEDIN_3_0
 int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *e);

test/ecdsatest.c

@@ -243,18 +243,6 @@ static int test_builtin(int n, int as)
 
     temp = ECDSA_size(eckey);
 
-    /*
-     * |as| indicates how we want to treat the key, i.e. what sort of
-     * computation we want to do with it.  The two choices are the key
-     * types EVP_PKEY_EC and EVP_PKEY_SM2.  It's perfectly possible to
-     * switch back and forth between those two key types, regardless of
-     * curve, even though the default is to have EVP_PKEY_SM2 for the
-     * SM2 curve and EVP_PKEY_EC for all other curves.
-     */
-    if (!TEST_true(EVP_PKEY_set_alias_type(pkey, as))
-        || !TEST_true(EVP_PKEY_set_alias_type(pkey_neg, as)))
-            goto err;
-
     if (!TEST_int_ge(temp, 0)
         || !TEST_ptr(sig = OPENSSL_malloc(sig_len = (size_t)temp))
         /* create a signature */

util/libcrypto.num

@@ -4393,7 +4393,7 @@ EVP_PKEY_get_raw_public_key             4518	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_get_raw_private_key            4519	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_asn1_set_get_priv_key          4520	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_asn1_set_get_pub_key           4521	3_0_0	EXIST::FUNCTION:
-EVP_PKEY_set_alias_type                 4522	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3_0
+EVP_PKEY_set_alias_type                 4522	3_0_0	NOEXIST::FUNCTION:DEPRECATEDIN_3_0
 RAND_keep_random_devices_open           4523	3_0_0	EXIST::FUNCTION:
 EC_POINT_set_compressed_coordinates     4524	3_0_0	EXIST::FUNCTION:EC
 EC_POINT_set_affine_coordinates         4525	3_0_0	EXIST::FUNCTION:EC