Make EVP_PKEY_asn1_add0() stricter about its input

It turns out that the strictness that was implemented in
EVP_PKEY_asn1_new() (see Github openssl/openssl#6880) was badly placed
for some usages, and that it's better to do this check only when the
method is getting registered.

Fixes #7758

Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7847)

(cherry picked from commit a86003162138031137727147c9b642d99db434b1)

CHANGES

@@ -9,7 +9,8 @@
 
  Changes between 1.0.2q and 1.0.2r [xx XXX xxxx]
 
-  *)
+  *) Move strictness check from EVP_PKEY_asn1_new() to EVP_PKEY_asn1_add0().
+     [Richard Levitte]
 
  Changes between 1.0.2p and 1.0.2q [20 Nov 2018]
 

crypto/asn1/ameth_lib.c

@@ -234,6 +234,21 @@ const EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_find_str(ENGINE **pe,
 
 int EVP_PKEY_asn1_add0(const EVP_PKEY_ASN1_METHOD *ameth)
 {
+    /*
+     * One of the following must be true:
+     *
+     * pem_str == NULL AND ASN1_PKEY_ALIAS is set
+     * pem_str != NULL AND ASN1_PKEY_ALIAS is clear
+     *
+     * Anything else is an error and may lead to a corrupt ASN1 method table
+     */
+    if (!((ameth->pem_str == NULL
+           && (ameth->pkey_flags & ASN1_PKEY_ALIAS) != 0)
+          || (ameth->pem_str != NULL
+              && (ameth->pkey_flags & ASN1_PKEY_ALIAS) == 0))) {
+        return 0;
+    }
+
     if (app_methods == NULL) {
         app_methods = sk_EVP_PKEY_ASN1_METHOD_new(ameth_cmp);
         if (!app_methods)
@@ -305,18 +320,6 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags,
     } else
         ameth->info = NULL;
 
-    /*
-     * One of the following must be true:
-     *
-     * pem_str == NULL AND ASN1_PKEY_ALIAS is set
-     * pem_str != NULL AND ASN1_PKEY_ALIAS is clear
-     *
-     * Anything else is an error and may lead to a corrupt ASN1 method table
-     */
-    if (!((pem_str == NULL && (flags & ASN1_PKEY_ALIAS) != 0)
-          || (pem_str != NULL && (flags & ASN1_PKEY_ALIAS) == 0)))
-        goto err;
-
     if (pem_str) {
         ameth->pem_str = BUF_strdup(pem_str);
         if (!ameth->pem_str)