Fixes for PKCS8 w/wo encryption as DER/ASN.1. Fixes for building with `--disable-oldnames`. Fix to enable the PKCS8 enc test without openssl comat. Added additional PKCS8 tests.

certs/include.am

@@ -13,6 +13,7 @@ EXTRA_DIST += \
 	     certs/client-relative-uri.pem \
 	     certs/ecc-key.pem \
 	     certs/ecc-privkey.pem \
+	     certs/ecc-privkeyPkcs8.der \
 	     certs/ecc-privkeyPkcs8.pem \
 	     certs/ecc-keyPkcs8Enc.pem \
 	     certs/ecc-keyPkcs8Enc.der \

src/ssl.c

@@ -4704,6 +4704,10 @@ int ProcessBuffer(WOLFSSL_CTX* ctx, const unsigned char* buff,
     int           devId = wolfSSL_CTX_GetDevId(ctx, ssl);
     word32        idx = 0;
     int           keySz = 0;
+#if (defined(WOLFSSL_ENCRYPTED_KEYS) && !defined(NO_PWDBASED)) || \
+     defined(HAVE_PKCS8)
+    word32        algId = 0;
+#endif
 #ifdef WOLFSSL_SMALL_STACK
     EncryptedInfo* info = NULL;
 #else
@@ -4734,7 +4738,7 @@ int ProcessBuffer(WOLFSSL_CTX* ctx, const unsigned char* buff,
 #endif
 
     XMEMSET(info, 0, sizeof(EncryptedInfo));
-#ifdef WOLFSSL_ENCRYPTED_KEYS
+#if defined(WOLFSSL_ENCRYPTED_KEYS) && !defined(NO_PWDBASED)
     if (ctx) {
         info->passwd_cb       = ctx->passwd_cb;
         info->passwd_userdata = ctx->passwd_userdata;
@@ -4771,7 +4775,6 @@ int ProcessBuffer(WOLFSSL_CTX* ctx, const unsigned char* buff,
         #ifdef HAVE_PKCS8
             /* if private key try and remove PKCS8 header */
             if (type == PRIVATEKEY_TYPE) {
-                word32 algId = 0;
                 if ((ret = ToTraditional_ex(der->buffer, der->length, &algId)) > 0) {
                     /* Found PKCS8 header */
                     /* ToTraditional_ex moves buff and returns adjusted length */
@@ -4799,46 +4802,48 @@ int ProcessBuffer(WOLFSSL_CTX* ctx, const unsigned char* buff,
         }
     }
 
-#ifdef WOLFSSL_ENCRYPTED_KEYS
-    /* for WOLFSSL_FILETYPE_PEM, PemToDer manage the decryption if required */
-    if (ret >= 0 && info->set && format != WOLFSSL_FILETYPE_PEM) {
-        /* decrypt */
+#if defined(WOLFSSL_ENCRYPTED_KEYS) && !defined(NO_PWDBASED)
+    /* for WOLFSSL_FILETYPE_PEM, PemToDer manages the decryption */
+    /* If private key type PKCS8 header wasn't already removed (algoId == 0) */
+    if (ret >= 0 && format != WOLFSSL_FILETYPE_PEM && info->passwd_cb &&
+        type == PRIVATEKEY_TYPE && algId == 0)
+    {
         int   passwordSz = NAME_SZ;
-#ifdef WOLFSSL_SMALL_STACK
-        char* password = NULL;
-#else
+    #ifndef WOLFSSL_SMALL_STACK
         char  password[NAME_SZ];
-#endif
-
-    #ifdef WOLFSSL_SMALL_STACK
-        password = (char*)XMALLOC(passwordSz, heap, DYNAMIC_TYPE_STRING);
-        if (password == NULL)
-            ret = MEMORY_E;
-        else
-    #endif
-        if (info->passwd_cb == NULL) {
-            WOLFSSL_MSG("No password callback set");
-            ret = NO_PASSWORD;
+    #else
+        char* password = (char*)XMALLOC(passwordSz, heap, DYNAMIC_TYPE_STRING);
+        if (password == NULL) {
+        #ifdef WOLFSSL_SMALL_STACK
+            XFREE(info, heap, DYNAMIC_TYPE_ENCRYPTEDINFO);
+        #endif
+            FreeDer(&der);
+            return MEMORY_E;
         }
-        else {
-            ret = info->passwd_cb(password, passwordSz, PEM_PASS_READ,
-                info->passwd_userdata);
-            if (ret >= 0) {
-                passwordSz = ret;
-
-                /* decrypt the key */
-                ret = wc_BufferKeyDecrypt(info, der->buffer, der->length,
-                    (byte*)password, passwordSz, WC_MD5);
+    #endif
+        /* get password */
+        ret = info->passwd_cb(password, passwordSz, PEM_PASS_READ,
+            info->passwd_userdata);
+        if (ret >= 0) {
+            passwordSz = ret;
 
-                ForceZero(password, passwordSz);
+            /* PKCS8 decrypt */
+            ret = ToTraditionalEnc(der->buffer, der->length,
+                                   password, passwordSz, &algId);
+            if (ret >= 0) {
+                der->length = ret;
             }
+            printf("ToTraditionalEnc ret %d\n", ret);
+            ret = 0; /* ignore failures and try parsing directly unencrypted */
+
+            ForceZero(password, passwordSz);
         }
 
     #ifdef WOLFSSL_SMALL_STACK
         XFREE(password, heap, DYNAMIC_TYPE_STRING);
     #endif
     }
-#endif /* WOLFSSL_ENCRYPTED_KEYS */
+#endif /* WOLFSSL_ENCRYPTED_KEYS && !NO_PWDBASED */
 
 #ifdef WOLFSSL_SMALL_STACK
     XFREE(info, heap, DYNAMIC_TYPE_ENCRYPTEDINFO);
@@ -4924,6 +4929,15 @@ int ProcessBuffer(WOLFSSL_CTX* ctx, const unsigned char* buff,
     }
 
     if (type == PRIVATEKEY_TYPE && format != WOLFSSL_FILETYPE_RAW) {
+    #if defined(WOLFSSL_ENCRYPTED_KEYS) || defined(HAVE_PKCS8)
+        /* attempt to detect key type */
+        if (algId == RSAk)
+            rsaKey = 1;
+        else if (algId == ECDSAk)
+            eccKey = 1;
+        else if (algId == ED25519k)
+            ed25519Key = 1;
+    #endif
     #ifndef NO_RSA
         if (!eccKey && !ed25519Key) {
             /* make sure RSA key can be used */

tests/api.c

@@ -4389,9 +4389,13 @@ static void test_wolfSSL_PKCS12(void)
 }
 
 
-#if (defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)) && \
-        !defined(NO_DES3) && !defined(NO_FILESYSTEM) && \
-        !defined(NO_ASN) && !defined(NO_PWDBASED) && !defined(NO_RSA)
+#if !defined(NO_FILESYSTEM) && !defined(NO_ASN) && defined(HAVE_PKCS8) && \
+    defined(WOLFSSL_ENCRYPTED_KEYS) && !defined(NO_DES3) && \
+    !defined(NO_PWDBASED) && !defined(NO_RSA)
+    #define TEST_PKCS8_ENC
+#endif
+
+#ifdef TEST_PKCS8_ENC
 /* for PKCS8 test case */
 static WC_INLINE int PKCS8TestCallBack(char* passwd, int sz, int rw, void* userdata)
 {
@@ -4406,15 +4410,14 @@ static WC_INLINE int PKCS8TestCallBack(char* passwd, int sz, int rw, void* userd
         case 1: /* flag set for specific WOLFSSL_CTX structure, note userdata
                  * can be anything the user wishes to be passed to the callback
                  * associated with the WOLFSSL_CTX */
-            strncpy(passwd, "yassl123", sz);
+            XSTRNCPY(passwd, "yassl123", sz);
             return 8;
 
-       default:
+        default:
             return BAD_FUNC_ARG;
     }
 }
-#endif
-
+#endif /* TEST_PKCS8_ENC */
 
 /* Testing functions dealing with PKCS8 */
 static void test_wolfSSL_PKCS8(void)
@@ -4422,7 +4425,9 @@ static void test_wolfSSL_PKCS8(void)
 #if !defined(NO_FILESYSTEM) && !defined(NO_ASN) && defined(HAVE_PKCS8)
     byte buffer[FOURK_BUF];
     byte der[FOURK_BUF];
-    const char eccPkcs8PrivKeyFile[] = "./certs/ecc-privkeyPkcs8.pem";
+    const char eccPkcs8PrivKeyPemFile[] = "./certs/ecc-privkeyPkcs8.pem";
+    const char eccPkcs8PrivKeyDerFile[] = "./certs/ecc-privkeyPkcs8.der";
+    const char serverKeyPkcs8PemFile[] = "./certs/server-keyPkcs8.pem";
     const char serverKeyPkcs8DerFile[] = "./certs/server-keyPkcs8.der";
     XFILE f;
     int bytes;
@@ -4432,12 +4437,14 @@ static void test_wolfSSL_PKCS8(void)
     ecc_key key;
     word32 x = 0;
 #endif
-#if (defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)) && \
-     defined(WOLFSSL_ENCRYPTED_KEYS) && !defined(NO_DES3) && \
-    !defined(NO_PWDBASED) && !defined(NO_RSA)
-    #define TEST_PKCS8_ENC
-    const char serverKeyPkcs8EncFile[] = "./certs/server-keyPkcs8Enc.pem";
-    int flag = 1;
+#ifdef TEST_PKCS8_ENC
+    const char serverKeyPkcs8EncPemFile[] = "./certs/server-keyPkcs8Enc.pem";
+    const char serverKeyPkcs8EncDerFile[] = "./certs/server-keyPkcs8Enc.der";
+    #ifdef HAVE_ECC
+        const char eccPkcs8EncPrivKeyPemFile[] = "./certs/ecc-keyPkcs8Enc.pem";
+        const char eccPkcs8EncPrivKeyDerFile[] = "./certs/ecc-keyPkcs8Enc.der";
+    #endif
+    int flag;
 #endif
 
     printf(testingFmt, "wolfSSL_PKCS8()");
@@ -4457,28 +4464,85 @@ static void test_wolfSSL_PKCS8(void)
 #endif
 
 #ifdef TEST_PKCS8_ENC
-    f = XFOPEN(serverKeyPkcs8EncFile, "rb");
+    wolfSSL_CTX_set_default_passwd_cb(ctx, PKCS8TestCallBack);
+    wolfSSL_CTX_set_default_passwd_cb_userdata(ctx, (void*)&flag);
+
+    /* test loading PEM PKCS8 encrypted file */
+    f = XFOPEN(serverKeyPkcs8EncPemFile, "rb");
     AssertTrue((f != XBADFILE));
     bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
     XFCLOSE(f);
+    flag = 1; /* used by password callback as return code */
+    AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_PEM), WOLFSSL_SUCCESS);
 
-    wolfSSL_CTX_set_default_passwd_cb(ctx, &PKCS8TestCallBack);
-    wolfSSL_CTX_set_default_passwd_cb_userdata(ctx, (void*)&flag);
+    /* this next case should fail because of password callback return code */
+    flag = 0; /* used by password callback as return code */
+    AssertIntNE(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_PEM), WOLFSSL_SUCCESS);
+
+    /* decrypt PKCS8 PEM to key in DER format with not using WOLFSSL_CTX */
+    AssertIntGT(wc_KeyPemToDer(buffer, bytes, der, (word32)sizeof(der),
+        "yassl123"), 0);
+
+    /* test that error value is returned with a bad password */
+    AssertIntLT(wc_KeyPemToDer(buffer, bytes, der, (word32)sizeof(der),
+        "bad"), 0);
+
+    /* test loading PEM PKCS8 encrypted file */
+    f = XFOPEN(serverKeyPkcs8EncDerFile, "rb");
+    AssertTrue((f != XBADFILE));
+    bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
+    XFCLOSE(f);
+    flag = 1; /* used by password callback as return code */
     AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
-                SSL_FILETYPE_PEM), SSL_SUCCESS);
+                WOLFSSL_FILETYPE_ASN1), WOLFSSL_SUCCESS);
 
-    /* this next case should fail if setting the user flag to a value other
-     * than 1 due to the password callback functions return value */
-    flag = 0;
-    wolfSSL_CTX_set_default_passwd_cb_userdata(ctx, (void*)&flag);
+    /* this next case should fail because of password callback return code */
+    flag = 0; /* used by password callback as return code */
+    AssertIntNE(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_ASN1), WOLFSSL_SUCCESS);
+
+    #ifdef HAVE_ECC
+    /* test loading PEM PKCS8 encrypted ECC Key file */
+    f = XFOPEN(eccPkcs8EncPrivKeyPemFile, "rb");
+    AssertTrue((f != XBADFILE));
+    bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
+    XFCLOSE(f);
+    flag = 1; /* used by password callback as return code */
+    AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_PEM), WOLFSSL_SUCCESS);
+
+    /* this next case should fail because of password callback return code */
+    flag = 0; /* used by password callback as return code */
     AssertIntNE(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
-                SSL_FILETYPE_PEM), SSL_SUCCESS);
+                WOLFSSL_FILETYPE_PEM), WOLFSSL_SUCCESS);
 
     /* decrypt PKCS8 PEM to key in DER format with not using WOLFSSL_CTX */
-    AssertIntGT(wc_KeyPemToDer(buffer, bytes, der, FOURK_BUF, "yassl123"), 0);
+    AssertIntGT(wc_KeyPemToDer(buffer, bytes, der, (word32)sizeof(der),
+        "yassl123"), 0);
 
     /* test that error value is returned with a bad password */
-    AssertIntLT(wc_KeyPemToDer(buffer, bytes, der, FOURK_BUF, "bad"), 0);
+    AssertIntLT(wc_KeyPemToDer(buffer, bytes, der, (word32)sizeof(der),
+        "bad"), 0);
+
+    /* test loading DER PKCS8 encrypted ECC Key file */
+    f = XFOPEN(eccPkcs8EncPrivKeyDerFile, "rb");
+    AssertTrue((f != XBADFILE));
+    bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
+    XFCLOSE(f);
+    flag = 1; /* used by password callback as return code */
+    AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_ASN1), WOLFSSL_SUCCESS);
+
+    /* this next case should fail because of password callback return code */
+    flag = 0; /* used by password callback as return code */
+    AssertIntNE(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_ASN1), WOLFSSL_SUCCESS);
+
+    /* leave flag as "okay" */
+    flag = 1;
+    #endif
 #endif /* TEST_PKCS8_ENC */
 
 
@@ -4488,20 +4552,31 @@ static void test_wolfSSL_PKCS8(void)
     bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
     XFCLOSE(f);
     AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
-                SSL_FILETYPE_ASN1), SSL_SUCCESS);
+                WOLFSSL_FILETYPE_ASN1), WOLFSSL_SUCCESS);
 
-    wolfSSL_CTX_free(ctx);
+    /* test loading PEM PKCS8 private key file (not encrypted) */
+    f = XFOPEN(serverKeyPkcs8PemFile, "rb");
+    AssertTrue((f != XBADFILE));
+    bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
+    XFCLOSE(f);
+    AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_PEM), WOLFSSL_SUCCESS);
 
 
     /* Test PKCS8 PEM ECC key no crypt */
-    f = XFOPEN(eccPkcs8PrivKeyFile, "rb");
+    f = XFOPEN(eccPkcs8PrivKeyPemFile, "rb");
     AssertTrue((f != XBADFILE));
     bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
     XFCLOSE(f);
 
-    /* decrypt PKCS8 PEM to key in DER format with not using WOLFSSL_CTX */
 #ifdef HAVE_ECC
-    AssertIntGT((bytes = wc_KeyPemToDer(buffer, bytes, der, FOURK_BUF, NULL)), 0);
+    /* Test PKCS8 PEM ECC key no crypt */
+    AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_PEM), WOLFSSL_SUCCESS);
+
+    /* decrypt PKCS8 PEM to key in DER format */
+    AssertIntGT((bytes = wc_KeyPemToDer(buffer, bytes, der,
+        (word32)sizeof(der), NULL)), 0);
     ret = wc_ecc_init(&key);
     if (ret == 0) {
         ret = wc_EccPrivateKeyDecode(der, &x, &key, bytes);
@@ -4509,10 +4584,24 @@ static void test_wolfSSL_PKCS8(void)
     }
     AssertIntEQ(ret, 0);
 #else
-    AssertIntEQ((bytes = wc_KeyPemToDer(buffer, bytes, der, FOURK_BUF, NULL)),
-        ASN_NO_PEM_HEADER);
+    AssertIntEQ((bytes = wc_KeyPemToDer(buffer, bytes, der,
+        (word32)sizeof(der), NULL)), ASN_NO_PEM_HEADER);
 #endif
 
+#ifdef HAVE_ECC
+    /* Test PKCS8 DER ECC key no crypt */
+    f = XFOPEN(eccPkcs8PrivKeyDerFile, "rb");
+    AssertTrue((f != XBADFILE));
+    bytes = (int)XFREAD(buffer, 1, sizeof(buffer), f);
+    XFCLOSE(f);
+
+    /* Test using a PKCS8 ECC PEM */
+    AssertIntEQ(wolfSSL_CTX_use_PrivateKey_buffer(ctx, buffer, bytes,
+                WOLFSSL_FILETYPE_ASN1), WOLFSSL_SUCCESS);
+#endif
+
+    wolfSSL_CTX_free(ctx);
+
     printf(resultFmt, passed);
 #endif /* !NO_FILESYSTEM && !NO_ASN && HAVE_PKCS8 */
 }