Merge pull request #5149 from julek-wolfssl/store-frags-v2

Re-use async to support WANT_WRITE while sending fragments

configure.ac

@@ -1453,7 +1453,7 @@ AC_ARG_ENABLE([lowresource],
 if test "$ENABLED_LOWRESOURCE" = "yes"
 then
     # low memory / flash flags
-    AM_CFLAGS="$AM_CFLAGS -DNO_SESSION_CACHE -DRSA_LOW_MEM -DGCM_SMALL -DCURVE25519_SMALL -DED25519_SMALL -DWOLFSSL_SMALL_CERT_VERIFY"
+    AM_CFLAGS="$AM_CFLAGS -DNO_SESSION_CACHE -DRSA_LOW_MEM -DGCM_SMALL -DCURVE25519_SMALL -DED25519_SMALL -DWOLFSSL_SMALL_CERT_VERIFY -DWOLFSSL_NO_ASYNC_IO"
 
     # low flash flags
     AM_CFLAGS="$AM_CFLAGS -DUSE_SLOW_SHA -DUSE_SLOW_SHA256 -DUSE_SLOW_SHA512"

examples/client/client.c

@@ -1055,7 +1055,8 @@ static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead,
             }
             else
         #endif
-            if (err != WOLFSSL_ERROR_WANT_READ && err != APP_DATA_READY) {
+            if (err != WOLFSSL_ERROR_WANT_READ &&
+                    err != WOLFSSL_ERROR_WANT_WRITE && err != APP_DATA_READY) {
                 fprintf(stderr, "SSL_read reply error %d, %s\n", err,
                                          wolfSSL_ERR_error_string(err, buffer));
                 if (!exitWithRet) {
@@ -1076,6 +1077,7 @@ static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead,
             }
         }
     } while ((mustRead && err == WOLFSSL_ERROR_WANT_READ)
+        || err == WOLFSSL_ERROR_WANT_WRITE
     #ifdef WOLFSSL_ASYNC_CRYPT
         || err == WC_PENDING_E
     #endif
@@ -2561,8 +2563,13 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                 break;
 
             case '6' :
+#ifdef WOLFSSL_ASYNC_IO
                 nonBlocking = 1;
                 simulateWantWrite = 1;
+#else
+                fprintf(stderr, "Ignoring -6 since async I/O support not "
+                                "compiled in.\n");
+#endif
                 break;
 
             case '7' :

examples/server/server.c

@@ -533,6 +533,8 @@ static void ServerRead(WOLFSSL* ssl, char* input, int inputLen)
             else
         #endif
             if (err != WOLFSSL_ERROR_WANT_READ
+                    && err != WOLFSSL_ERROR_WANT_WRITE /* Can happen during
+                                                        * handshake */
         #ifdef HAVE_SECURE_RENEGOTIATION
                     && err != APP_DATA_READY
         #endif
@@ -2049,8 +2051,13 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                 break;
 
             case '6' :
+#ifdef WOLFSSL_ASYNC_IO
                 nonBlocking = 1;
                 simulateWantWrite = 1;
+#else
+                fprintf(stderr, "Ignoring -6 since async I/O support not "
+                                "compiled in.\n");
+#endif
                 break;
             case '7' :
                 minVersion = atoi(myoptarg);

src/sniffer.c

@@ -2332,9 +2332,8 @@ static int SetupKeys(const byte* input, int* sslBytes, SnifferSession* session,
     WOLFSSL* ssl = session->sslServer;
 
 #ifdef WOLFSSL_ASYNC_CRYPT
-    SetupKeysArgs* args = (SetupKeysArgs*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
+    SetupKeysArgs* args = NULL;
+    WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
 #else
     SetupKeysArgs  args[1];
 #endif
@@ -2349,6 +2348,15 @@ static int SetupKeys(const byte* input, int* sslBytes, SnifferSession* session,
     }
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    if (ssl->async == NULL) {
+        ssl->async = (struct WOLFSSL_ASYNC*)
+                XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                        DYNAMIC_TYPE_ASYNC);
+        if (ssl->async == NULL)
+            ERROR_OUT(MEMORY_E, exit_sk);
+    }
+    args = (SetupKeysArgs*)ssl->async->args;
+
     ret = wolfSSL_AsyncPop(ssl, &ssl->options.asyncState);
     if (ret != WC_NOT_PENDING_E) {
         /* Check for error */
@@ -2363,7 +2371,7 @@ static int SetupKeys(const byte* input, int* sslBytes, SnifferSession* session,
         ssl->options.asyncState = TLS_ASYNC_BEGIN;
         XMEMSET(args, 0, sizeof(SetupKeysArgs));
     #ifdef WOLFSSL_ASYNC_CRYPT
-        ssl->async.freeArgs = FreeSetupKeysArgs;
+        ssl->async->freeArgs = FreeSetupKeysArgs;
     #endif
     #ifdef WOLFSSL_ASYNC_CRYPT
         args->key = (SnifferKey*)XMALLOC(sizeof(SnifferKey), NULL,
@@ -3090,7 +3098,11 @@ exit_sk:
 #endif
 
     /* Final cleanup */
+#ifdef WOLFSSL_ASYNC_CRYPT
+    FreeAsyncCtx(ssl, 1);
+#else
     FreeSetupKeysArgs(ssl, args);
+#endif
 
     return ret;
 }

src/ssl.c

@@ -11778,6 +11778,9 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
     #if !(defined(WOLFSSL_NO_TLS12) && defined(NO_OLD_TLS) && defined(WOLFSSL_TLS13))
         int neededState;
     #endif
+        int ret = 0;
+
+        (void)ret;
 
         WOLFSSL_ENTER("SSL_connect()");
 
@@ -11820,14 +11823,16 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
             if ((ssl->ConnectFilter(ssl, ssl->ConnectFilter_arg, &res) ==
                  WOLFSSL_SUCCESS) &&
                 (res == WOLFSSL_NETFILTER_REJECT)) {
-                WOLFSSL_ERROR(ssl->error = SOCKET_FILTERED_E);
+                ssl->error = SOCKET_FILTERED_E;
+                WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
         }
 #endif /* WOLFSSL_WOLFSENTRY_HOOKS */
 
         if (ssl->options.side != WOLFSSL_CLIENT_END) {
-            WOLFSSL_ERROR(ssl->error = SIDE_ERROR);
+            ssl->error = SIDE_ERROR;
+            WOLFSSL_ERROR(ssl->error);
             return WOLFSSL_FATAL_ERROR;
         }
 
@@ -11846,11 +11851,11 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
             && ssl->error != WC_PENDING_E
         #endif
         ) {
-            if ( (ssl->error = SendBuffered(ssl)) == 0) {
+            if ( (ret = SendBuffered(ssl)) == 0) {
                 /* fragOffset is non-zero when sending fragments. On the last
                  * fragment, fragOffset is zero again, and the state can be
                  * advanced. */
-                if (ssl->fragOffset == 0) {
+                if (ssl->fragOffset == 0 && !ssl->options.buildingMsg) {
                     if (ssl->options.connectState == CONNECT_BEGIN ||
                         ssl->options.connectState == HELLO_AGAIN ||
                        (ssl->options.connectState >= FIRST_REPLY_DONE &&
@@ -11859,6 +11864,10 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                         WOLFSSL_MSG("connect state: "
                                     "Advanced from last buffered fragment send");
                     }
+                #ifdef WOLFSSL_ASYNC_IO
+                    /* Cleanup async */
+                    FreeAsyncCtx(ssl, 0);
+                #endif
                 }
                 else {
                     WOLFSSL_MSG("connect state: "
@@ -11866,11 +11875,19 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                 }
             }
             else {
+                ssl->error = ret;
                 WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
         }
 
+        ret = RetrySendAlert(ssl);
+        if (ret != 0) {
+            ssl->error = ret;
+            WOLFSSL_ERROR(ssl->error);
+            return WOLFSSL_FATAL_ERROR;
+        }
+
         switch (ssl->options.connectState) {
 
         case CONNECT_BEGIN :
@@ -12108,6 +12125,12 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                 ssl->secure_renegotiation->startScr = 0;
             }
         #endif /* WOLFSSL_ASYNC_CRYPT && HAVE_SECURE_RENEGOTIATION */
+        #if defined(WOLFSSL_ASYNC_IO) && !defined(WOLFSSL_ASYNC_CRYPT)
+            /* Free the remaining async context if not using it for crypto */
+            FreeAsyncCtx(ssl, 1);
+        #endif
+
+            ssl->error = 0; /* clear the error */
 
             WOLFSSL_LEAVE("SSL_connect()", WOLFSSL_SUCCESS);
             return WOLFSSL_SUCCESS;
@@ -12199,6 +12222,9 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
         word16 haveAnon = 0;
         word16 haveMcast = 0;
 #endif
+        int ret = 0;
+
+        (void)ret;
 
         if (ssl == NULL)
             return WOLFSSL_FATAL_ERROR;
@@ -12230,7 +12256,8 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
             if ((ssl->AcceptFilter(ssl, ssl->AcceptFilter_arg, &res) ==
                  WOLFSSL_SUCCESS) &&
                 (res == WOLFSSL_NETFILTER_REJECT)) {
-                WOLFSSL_ERROR(ssl->error = SOCKET_FILTERED_E);
+                ssl->error = SOCKET_FILTERED_E;
+                WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
         }
@@ -12256,7 +12283,8 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
         (void)haveMcast;
 
         if (ssl->options.side != WOLFSSL_SERVER_END) {
-            WOLFSSL_ERROR(ssl->error = SIDE_ERROR);
+            ssl->error = SIDE_ERROR;
+            WOLFSSL_ERROR(ssl->error);
             return WOLFSSL_FATAL_ERROR;
         }
 
@@ -12295,7 +12323,8 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                 #endif
                     {
                         WOLFSSL_MSG("accept error: server key required");
-                        WOLFSSL_ERROR(ssl->error = NO_PRIVATE_KEY);
+                        ssl->error = NO_PRIVATE_KEY;
+                        WOLFSSL_ERROR(ssl->error);
                         return WOLFSSL_FATAL_ERROR;
                     }
                 }
@@ -12318,11 +12347,11 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
             && ssl->error != WC_PENDING_E
         #endif
         ) {
-            if ( (ssl->error = SendBuffered(ssl)) == 0) {
+            if ( (ret = SendBuffered(ssl)) == 0) {
                 /* fragOffset is non-zero when sending fragments. On the last
                  * fragment, fragOffset is zero again, and the state can be
                  * advanced. */
-                if (ssl->fragOffset == 0) {
+                if (ssl->fragOffset == 0 && !ssl->options.buildingMsg) {
                     if (ssl->options.acceptState == ACCEPT_FIRST_REPLY_DONE ||
                         ssl->options.acceptState == SERVER_HELLO_SENT ||
                         ssl->options.acceptState == CERT_SENT ||
@@ -12336,6 +12365,10 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                         WOLFSSL_MSG("accept state: "
                                     "Advanced from last buffered fragment send");
                     }
+                #ifdef WOLFSSL_ASYNC_IO
+                    /* Cleanup async */
+                    FreeAsyncCtx(ssl, 0);
+                #endif
                 }
                 else {
                     WOLFSSL_MSG("accept state: "
@@ -12343,11 +12376,19 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                 }
             }
             else {
+                ssl->error = ret;
                 WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
         }
 
+        ret = RetrySendAlert(ssl);
+        if (ret != 0) {
+            ssl->error = ret;
+            WOLFSSL_ERROR(ssl->error);
+            return WOLFSSL_FATAL_ERROR;
+        }
+
         switch (ssl->options.acceptState) {
 
         case ACCEPT_BEGIN :
@@ -12563,6 +12604,10 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                 ssl->secure_renegotiation->startScr = 0;
             }
 #endif /* WOLFSSL_ASYNC_CRYPT && HAVE_SECURE_RENEGOTIATION */
+#if defined(WOLFSSL_ASYNC_IO) && !defined(WOLFSSL_ASYNC_CRYPT)
+            /* Free the remaining async context if not using it for crypto */
+            FreeAsyncCtx(ssl, 1);
+#endif
 
 #if defined(WOLFSSL_SESSION_EXPORT) && defined(WOLFSSL_DTLS)
             if (ssl->dtls_export) {
@@ -12573,6 +12618,7 @@ int wolfSSL_DTLS_SetCookieSecret(WOLFSSL* ssl,
                 }
             }
 #endif
+            ssl->error = 0; /* clear the error */
 
             WOLFSSL_LEAVE("SSL_accept()", WOLFSSL_SUCCESS);
             return WOLFSSL_SUCCESS;

src/tls.c

@@ -4683,6 +4683,7 @@ int TLSX_ValidateSupportedCurves(WOLFSSL* ssl, byte first, byte second) {
                         defOid = 0;
                         defSz = 80;
                     }
+                    key |= ssl->pkCurveOID == oid;
                 break;
     #endif /* HAVE_ECC && WOLFSSL_STATIC_DH */
 #endif

src/tls13.c

@@ -2330,17 +2330,23 @@ int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input,
     int ret;
     BuildMsg13Args* args;
     BuildMsg13Args  lcl_args;
-#ifdef WOLFSSL_ASYNC_CRYPT
-    args = (BuildMsg13Args*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
-#endif
 
     WOLFSSL_ENTER("BuildTls13Message");
 
 #ifdef WOLFSSL_ASYNC_CRYPT
     ret = WC_NOT_PENDING_E;
     if (asyncOkay) {
+        WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
+
+        if (ssl->async == NULL) {
+            ssl->async = (struct WOLFSSL_ASYNC*)
+                    XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                            DYNAMIC_TYPE_ASYNC);
+            if (ssl->async == NULL)
+                return MEMORY_E;
+        }
+        args = (BuildMsg13Args*)ssl->async->args;
+
         ret = wolfSSL_AsyncPop(ssl, &ssl->options.buildMsgState);
         if (ret != WC_NOT_PENDING_E) {
             /* Check for error */
@@ -2367,7 +2373,8 @@ int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input,
         args->idx  = RECORD_HEADER_SZ;
         args->headerSz = RECORD_HEADER_SZ;
     #ifdef WOLFSSL_ASYNC_CRYPT
-        ssl->async.freeArgs = FreeBuildMsg13Args;
+        if (asyncOkay)
+            ssl->async->freeArgs = FreeBuildMsg13Args;
     #endif
     }
 
@@ -2475,10 +2482,12 @@ exit_buildmsg:
         ret = args->sz;
 
     /* Final cleanup */
-    FreeBuildMsg13Args(ssl, args);
 #ifdef WOLFSSL_ASYNC_CRYPT
-    ssl->async.freeArgs = NULL;
+    if (asyncOkay)
+        FreeAsyncCtx(ssl, 0);
+    else
 #endif
+        FreeBuildMsg13Args(ssl, args);
 
     return ret;
 }
@@ -3055,9 +3064,8 @@ int SendTls13ClientHello(WOLFSSL* ssl)
 {
     int ret;
 #ifdef WOLFSSL_ASYNC_CRYPT
-    Sch13Args* args = (Sch13Args*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
+    Sch13Args* args = NULL;
+    WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
 #else
     Sch13Args  args[1];
 #endif
@@ -3094,6 +3102,16 @@ int SendTls13ClientHello(WOLFSSL* ssl)
     }
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    if (ssl->async == NULL) {
+        ssl->async = (struct WOLFSSL_ASYNC*)
+                XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                        DYNAMIC_TYPE_ASYNC);
+        if (ssl->async == NULL)
+            return MEMORY_E;
+        ssl->async->freeArgs = NULL;
+    }
+    args = (Sch13Args*)ssl->async->args;
+
     ret = wolfSSL_AsyncPop(ssl, &ssl->options.asyncState);
     if (ret != WC_NOT_PENDING_E) {
         /* Check for error */
@@ -3272,6 +3290,12 @@ int SendTls13ClientHello(WOLFSSL* ssl)
 
     ssl->buffers.outputBuffer.length += args->sendSz;
 
+    /* Advance state and proceed */
+    ssl->options.asyncState = TLS_ASYNC_END;
+    /* case TLS_ASYNC_BUILD */
+    FALL_THROUGH;
+
+    case TLS_ASYNC_END:
 #ifdef WOLFSSL_EARLY_DATA_GROUP
     if (ssl->earlyData == no_early_data)
 #endif
@@ -3283,6 +3307,11 @@ int SendTls13ClientHello(WOLFSSL* ssl)
         ret = INPUT_CASE_ERROR;
     } /* switch (ssl->options.asyncState) */
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    if (ret == 0)
+        FreeAsyncCtx(ssl, 0);
+#endif
+
     WOLFSSL_LEAVE("SendTls13ClientHello", ret);
     WOLFSSL_END(WC_FUNC_CLIENT_HELLO_SEND);
 
@@ -3317,9 +3346,8 @@ int DoTls13ServerHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
     int ret;
     byte suite[2];
 #ifdef WOLFSSL_ASYNC_CRYPT
-    Dsh13Args* args = (Dsh13Args*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
+    Dsh13Args* args = NULL;
+    WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
 #else
     Dsh13Args  args[1];
 #endif
@@ -3331,6 +3359,16 @@ int DoTls13ServerHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
         return BAD_FUNC_ARG;
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    if (ssl->async == NULL) {
+        ssl->async = (struct WOLFSSL_ASYNC*)
+                XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                        DYNAMIC_TYPE_ASYNC);
+        if (ssl->async == NULL)
+            return MEMORY_E;
+        ssl->async->freeArgs = NULL;
+    }
+    args = (Dsh13Args*)ssl->async->args;
+
     ret = wolfSSL_AsyncPop(ssl, &ssl->options.asyncState);
     if (ret != WC_NOT_PENDING_E) {
         /* Check for error */
@@ -3670,6 +3708,11 @@ int DoTls13ServerHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
         ret = INPUT_CASE_ERROR;
     } /* switch (ssl->options.asyncState) */
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    if (ret == 0)
+        FreeAsyncCtx(ssl, 0);
+#endif
+
     WOLFSSL_LEAVE("DoTls13ServerHello", ret);
     WOLFSSL_END(WC_FUNC_SERVER_HELLO_DO);
 
@@ -4614,9 +4657,8 @@ int DoTls13ClientHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
 {
     int ret;
 #ifdef WOLFSSL_ASYNC_CRYPT
-    Dch13Args* args = (Dch13Args*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
+    Dch13Args* args = NULL;
+    WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
 #else
     Dch13Args  args[1];
 #endif
@@ -4625,6 +4667,15 @@ int DoTls13ClientHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
     WOLFSSL_ENTER("DoTls13ClientHello");
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    if (ssl->async == NULL) {
+        ssl->async = (struct WOLFSSL_ASYNC*)
+                XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                        DYNAMIC_TYPE_ASYNC);
+        if (ssl->async == NULL)
+            ERROR_OUT(MEMORY_E, exit_dch);
+    }
+    args = (Dch13Args*)ssl->async->args;
+
     ret = wolfSSL_AsyncPop(ssl, &ssl->options.asyncState);
     if (ret != WC_NOT_PENDING_E) {
         /* Check for error */
@@ -4640,7 +4691,7 @@ int DoTls13ClientHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
         ssl->options.asyncState = TLS_ASYNC_BEGIN;
         XMEMSET(args, 0, sizeof(Dch13Args));
     #ifdef WOLFSSL_ASYNC_CRYPT
-        ssl->async.freeArgs = FreeDch13Args;
+        ssl->async->freeArgs = FreeDch13Args;
     #endif
     }
 
@@ -4971,6 +5022,9 @@ exit_dch:
 #endif
 
     FreeDch13Args(ssl, args);
+#ifdef WOLFSSL_ASYNC_CRYPT
+    FreeAsyncCtx(ssl, 0);
+#endif
     WOLFSSL_END(WC_FUNC_CLIENT_HELLO_DO);
 
     return ret;
@@ -5878,6 +5932,8 @@ static int SendTls13Certificate(WOLFSSL* ssl)
         word32 i = RECORD_HEADER_SZ;
         int    sendSz = RECORD_HEADER_SZ;
 
+        ssl->options.buildingMsg = 1;
+
         if (ssl->fragOffset == 0) {
             if (headerSz + certSz + extSz + certChainSz <=
                                             maxFragment - HANDSHAKE_HEADER_SZ) {
@@ -5999,6 +6055,7 @@ static int SendTls13Certificate(WOLFSSL* ssl)
 
     if (ret != WANT_WRITE) {
         /* Clean up the fragment offset. */
+        ssl->options.buildingMsg = 0;
         ssl->fragOffset = 0;
         if (ssl->options.side == WOLFSSL_SERVER_END)
             ssl->options.serverState = SERVER_CERT_COMPLETE;
@@ -6061,9 +6118,8 @@ static int SendTls13CertificateVerify(WOLFSSL* ssl)
     int ret = 0;
     buffer* sig = &ssl->buffers.sig;
 #ifdef WOLFSSL_ASYNC_CRYPT
-    Scv13Args* args = (Scv13Args*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
+    Scv13Args* args = NULL;
+    WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
 #else
     Scv13Args  args[1];
 #endif
@@ -6072,6 +6128,15 @@ static int SendTls13CertificateVerify(WOLFSSL* ssl)
     WOLFSSL_ENTER("SendTls13CertificateVerify");
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    if (ssl->async == NULL) {
+        ssl->async = (struct WOLFSSL_ASYNC*)
+                XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                        DYNAMIC_TYPE_ASYNC);
+        if (ssl->async == NULL)
+            ERROR_OUT(MEMORY_E, exit_scv);
+    }
+    args = (Scv13Args*)ssl->async->args;
+
     ret = wolfSSL_AsyncPop(ssl, &ssl->options.asyncState);
     if (ret != WC_NOT_PENDING_E) {
         /* Check for error */
@@ -6086,7 +6151,7 @@ static int SendTls13CertificateVerify(WOLFSSL* ssl)
         ssl->options.asyncState = TLS_ASYNC_BEGIN;
         XMEMSET(args, 0, sizeof(Scv13Args));
     #ifdef WOLFSSL_ASYNC_CRYPT
-        ssl->async.freeArgs = FreeScv13Args;
+        ssl->async->freeArgs = FreeScv13Args;
     #endif
     }
 
@@ -6442,6 +6507,10 @@ exit_scv:
     /* Final cleanup */
     FreeScv13Args(ssl, args);
     FreeKeyExchange(ssl);
+#ifdef WOLFSSL_ASYNC_IO
+    /* Cleanup async */
+    FreeAsyncCtx(ssl, 0);
+#endif
 
     return ret;
 }
@@ -6537,9 +6606,8 @@ static int DoTls13CertificateVerify(WOLFSSL* ssl, byte* input,
     int         ret = 0;
     buffer*     sig = &ssl->buffers.sig;
 #ifdef WOLFSSL_ASYNC_CRYPT
-    Dcv13Args* args = (Dcv13Args*)ssl->async.args;
-    typedef char args_test[sizeof(ssl->async.args) >= sizeof(*args) ? 1 : -1];
-    (void)sizeof(args_test);
+    Dcv13Args* args = NULL;
+    WOLFSSL_ASSERT_SIZEOF_GE(ssl->async->args, *args);
 #else
     Dcv13Args  args[1];
 #endif
@@ -6548,6 +6616,15 @@ static int DoTls13CertificateVerify(WOLFSSL* ssl, byte* input,
     WOLFSSL_ENTER("DoTls13CertificateVerify");
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    if (ssl->async == NULL) {
+        ssl->async = (struct WOLFSSL_ASYNC*)
+                XMALLOC(sizeof(struct WOLFSSL_ASYNC), ssl->heap,
+                        DYNAMIC_TYPE_ASYNC);
+        if (ssl->async == NULL)
+            ERROR_OUT(MEMORY_E, exit_dcv);
+    }
+    args = (Dcv13Args*)ssl->async->args;
+
     ret = wolfSSL_AsyncPop(ssl, &ssl->options.asyncState);
     if (ret != WC_NOT_PENDING_E) {
         /* Check for error */
@@ -6566,7 +6643,7 @@ static int DoTls13CertificateVerify(WOLFSSL* ssl, byte* input,
         args->idx = *inOutIdx;
         args->begin = *inOutIdx;
     #ifdef WOLFSSL_ASYNC_CRYPT
-        ssl->async.freeArgs = FreeDcv13Args;
+        ssl->async->freeArgs = FreeDcv13Args;
     #endif
     }
 
@@ -6926,6 +7003,10 @@ exit_dcv:
     /* Final cleanup */
     FreeDcv13Args(ssl, args);
     FreeKeyExchange(ssl);
+#ifdef WOLFSSL_ASYNC_IO
+    /* Cleanup async */
+    FreeAsyncCtx(ssl, 0);
+#endif
 
     return ret;
 }
@@ -8473,7 +8554,7 @@ int DoTls13HandShakeMsgType(WOLFSSL* ssl, byte* input, word32* inOutIdx,
 #endif /* NO_WOLFSSL_SERVER */
     }
 
-#if defined(WOLFSSL_ASYNC_CRYPT) || defined(WOLFSSL_NONBLOCK_OCSP)
+#if defined(WOLFSSL_ASYNC_CRYPT) || defined(WOLFSSL_ASYNC_IO)
     /* if async, offset index so this msg will be processed again */
     if ((ret == WC_PENDING_E || ret == OCSP_WANT_READ) && *inOutIdx > 0) {
         *inOutIdx -= HANDSHAKE_HEADER_SZ;
@@ -8608,6 +8689,8 @@ int DoTls13HandShakeMsg(WOLFSSL* ssl, byte* input, word32* inOutIdx,
  */
 int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
 {
+    int ret = 0;
+
     WOLFSSL_ENTER("wolfSSL_connect_TLSv13()");
 
     #ifdef HAVE_ERRNO_H
@@ -8615,7 +8698,8 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
     #endif
 
     if (ssl->options.side != WOLFSSL_CLIENT_END) {
-        WOLFSSL_ERROR(ssl->error = SIDE_ERROR);
+        ssl->error = SIDE_ERROR;
+        WOLFSSL_ERROR(ssl->error);
         return WOLFSSL_FATAL_ERROR;
     }
 
@@ -8625,7 +8709,8 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
         if ((ssl->ConnectFilter(ssl, ssl->ConnectFilter_arg, &res) ==
              WOLFSSL_SUCCESS) &&
             (res == WOLFSSL_NETFILTER_REJECT)) {
-            WOLFSSL_ERROR(ssl->error = SOCKET_FILTERED_E);
+            ssl->error = SOCKET_FILTERED_E;
+            WOLFSSL_ERROR(ssl->error);
             return WOLFSSL_FATAL_ERROR;
         }
     }
@@ -8638,11 +8723,11 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
         && ssl->error != WC_PENDING_E
     #endif
     ) {
-        if ((ssl->error = SendBuffered(ssl)) == 0) {
+        if ((ret = SendBuffered(ssl)) == 0) {
             /* fragOffset is non-zero when sending fragments. On the last
              * fragment, fragOffset is zero again, and the state can be
              * advanced. */
-            if (ssl->fragOffset == 0) {
+            if (ssl->fragOffset == 0 && !ssl->options.buildingMsg) {
                 /* Only increment from states in which we send data */
                 if (ssl->options.connectState == CONNECT_BEGIN ||
                     ssl->options.connectState == HELLO_AGAIN ||
@@ -8657,13 +8742,24 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
                 WOLFSSL_MSG("connect state: "
                             "Not advanced, more fragments to send");
             }
+        #ifdef WOLFSSL_ASYNC_IO
+            FreeAsyncCtx(ssl, 0);
+        #endif
         }
         else {
+            ssl->error = ret;
             WOLFSSL_ERROR(ssl->error);
             return WOLFSSL_FATAL_ERROR;
         }
     }
 
+    ret = RetrySendAlert(ssl);
+    if (ret != 0) {
+        ssl->error = ret;
+        WOLFSSL_ERROR(ssl->error);
+        return WOLFSSL_FATAL_ERROR;
+    }
+
     switch (ssl->options.connectState) {
 
         case CONNECT_BEGIN:
@@ -8857,6 +8953,12 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
             if (!ssl->options.keepResources) {
                 FreeHandshakeResources(ssl);
             }
+        #if defined(WOLFSSL_ASYNC_IO) && !defined(WOLFSSL_ASYNC_CRYPT)
+            /* Free the remaining async context if not using it for crypto */
+            FreeAsyncCtx(ssl, 1);
+        #endif
+
+            ssl->error = 0; /* clear the error */
 
             WOLFSSL_LEAVE("wolfSSL_connect_TLSv13()", WOLFSSL_SUCCESS);
             return WOLFSSL_SUCCESS;
@@ -9521,6 +9623,7 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
 #if !defined(NO_CERTS) && (defined(HAVE_SESSION_TICKET) || !defined(NO_PSK))
     word16 havePSK = 0;
 #endif
+    int ret = 0;
     WOLFSSL_ENTER("SSL_accept_TLSv13()");
 
 #ifdef HAVE_ERRNO_H
@@ -9532,7 +9635,8 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
 #endif
 
     if (ssl->options.side != WOLFSSL_SERVER_END) {
-        WOLFSSL_ERROR(ssl->error = SIDE_ERROR);
+        ssl->error = SIDE_ERROR;
+        WOLFSSL_ERROR(ssl->error);
         return WOLFSSL_FATAL_ERROR;
     }
 
@@ -9542,7 +9646,8 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
         if ((ssl->AcceptFilter(ssl, ssl->AcceptFilter_arg, &res) ==
              WOLFSSL_SUCCESS) &&
             (res == WOLFSSL_NETFILTER_REJECT)) {
-            WOLFSSL_ERROR(ssl->error = SOCKET_FILTERED_E);
+            ssl->error = SOCKET_FILTERED_E;
+            WOLFSSL_ERROR(ssl->error);
             return WOLFSSL_FATAL_ERROR;
         }
     }
@@ -9566,7 +9671,8 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 !ssl->buffers.certificate->buffer) {
 
                 WOLFSSL_MSG("accept error: server cert required");
-                WOLFSSL_ERROR(ssl->error = NO_PRIVATE_KEY);
+                ssl->error = NO_PRIVATE_KEY;
+                WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
 
@@ -9584,7 +9690,8 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
             #endif
                 {
                     WOLFSSL_MSG("accept error: server key required");
-                    WOLFSSL_ERROR(ssl->error = NO_PRIVATE_KEY);
+                    ssl->error = NO_PRIVATE_KEY;
+                    WOLFSSL_ERROR(ssl->error);
                     return WOLFSSL_FATAL_ERROR;
                 }
             }
@@ -9599,11 +9706,11 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
         && ssl->error != WC_PENDING_E
     #endif
     ) {
-        if ((ssl->error = SendBuffered(ssl)) == 0) {
+        if ((ret = SendBuffered(ssl)) == 0) {
             /* fragOffset is non-zero when sending fragments. On the last
              * fragment, fragOffset is zero again, and the state can be
              * advanced. */
-            if (ssl->fragOffset == 0) {
+            if (ssl->fragOffset == 0 && !ssl->options.buildingMsg) {
                 /* Only increment from states in which we send data */
                 if (ssl->options.acceptState == TLS13_ACCEPT_CLIENT_HELLO_DONE ||
                     ssl->options.acceptState == TLS13_ACCEPT_HELLO_RETRY_REQUEST_DONE ||
@@ -9620,6 +9727,9 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                     WOLFSSL_MSG("accept state: "
                                 "Advanced from last buffered fragment send");
                 }
+            #ifdef WOLFSSL_ASYNC_IO
+                FreeAsyncCtx(ssl, 0);
+            #endif
             }
             else {
                 WOLFSSL_MSG("accept state: "
@@ -9627,11 +9737,19 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
             }
         }
         else {
+            ssl->error = ret;
             WOLFSSL_ERROR(ssl->error);
             return WOLFSSL_FATAL_ERROR;
         }
     }
 
+    ret = RetrySendAlert(ssl);
+    if (ret != 0) {
+        ssl->error = ret;
+        WOLFSSL_ERROR(ssl->error);
+        return WOLFSSL_FATAL_ERROR;
+    }
+
     switch (ssl->options.acceptState) {
 
 #ifdef HAVE_SECURE_RENEGOTIATION
@@ -9887,6 +10005,13 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 FreeHandshakeResources(ssl);
             }
 
+#if defined(WOLFSSL_ASYNC_IO) && !defined(WOLFSSL_ASYNC_CRYPT)
+            /* Free the remaining async context if not using it for crypto */
+            FreeAsyncCtx(ssl, 1);
+#endif
+
+            ssl->error = 0; /* clear the error */
+
             WOLFSSL_LEAVE("SSL_accept()", WOLFSSL_SUCCESS);
             return WOLFSSL_SUCCESS;
 

tests/test-dtls-mtu.conf

@@ -3521,3 +3521,17 @@
 -f
 -v 3
 -l AES256-SHA256
+
+# server DTLSv1.2 TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 with write blocking
+-u 512
+-6
+-f
+-v 3
+-l TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+
+# client DTLSv1.2 TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 with write blocking
+-u 512
+-6
+-f
+-v 3
+-l TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256

tests/test-dtls-reneg-server.conf

@@ -1061,3 +1061,19 @@
 -a
 -v 2
 -l ADH-AES128-SHA
+
+# server DTLSv1.2 TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 with non-blocking write
+-m
+-u 512
+-v 3
+-6
+-l TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+-f
+
+# client DTLSv1.2 TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 with non-blocking write
+-R
+-u 512
+-v 3
+-6
+-l TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+-f

tests/test-maxfrag.conf

@@ -177,3 +177,14 @@
 -v 3
 -l DHE-RSA-AES256-GCM-SHA384
 -F 6
+
+# server TLSv1.2 DHE-RSA-AES256-GCM-SHA384
+-v 3
+-l DHE-RSA-AES256-GCM-SHA384
+-6
+
+# client TLSv1.2 DHE-RSA-AES256-GCM-SHA384
+-v 3
+-l DHE-RSA-AES256-GCM-SHA384
+-F 1
+-6

tests/test.conf

@@ -2070,3 +2070,13 @@
 -v 3
 -l ECDHE-RSA-AES128-SHA256
 
+# server test with a blocking write socket
+-v 3
+-6
+-l TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+
+# client test with a blocking write socket
+-v 3
+-6
+-l TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+

wolfssl/internal.h

@@ -1613,6 +1613,9 @@ enum Misc {
 
 #define MAX_ENCRYPT_SZ ENCRYPT_LEN
 
+#define WOLFSSL_ASSERT_SIZEOF_GE(x, y)                              \
+    typedef char _args_test[sizeof((x)) >= sizeof((y)) ? 1 : -1];    \
+    (void)sizeof(_args_test)
 
 /* states */
 enum states {
@@ -1749,9 +1752,12 @@ WOLFSSL_LOCAL int GetPrivateKeySigSize(WOLFSSL* ssl);
     WOLFSSL_LOCAL int  InitSigPkCb(WOLFSSL* ssl, SignatureCtx* sigCtx);
 #endif
 #endif
+#ifdef WOLFSSL_ASYNC_IO
+WOLFSSL_LOCAL void FreeAsyncCtx(WOLFSSL* ssl, byte freeAsync);
+#endif
 WOLFSSL_LOCAL void FreeKeyExchange(WOLFSSL* ssl);
 WOLFSSL_LOCAL void FreeSuites(WOLFSSL* ssl);
-WOLFSSL_LOCAL int  ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx, word32 size);
+WOLFSSL_LOCAL int  ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx, word32 totalSz);
 WOLFSSL_LOCAL int  MatchDomainName(const char* pattern, int len, const char* str);
 #ifndef NO_CERTS
 WOLFSSL_LOCAL int  CheckForAltNames(DecodedCert* dCert, const char* domain, int* checkCN);
@@ -3507,7 +3513,6 @@ enum AcceptState {
     ACCEPT_HELLO_RETRY_REQUEST_DONE,
     ACCEPT_FIRST_REPLY_DONE,
     SERVER_HELLO_SENT,
-    SERVER_EXTENSIONS_SENT,
     CERT_SENT,
     CERT_VERIFY_SENT,
     CERT_STATUS_SENT,
@@ -3695,7 +3700,6 @@ typedef struct Options {
     word16            usingPSK_cipher:1;  /* are using psk as cipher */
     word16            usingAnon_cipher:1; /* are we using an anon cipher */
     word16            noPskDheKe:1;       /* Don't use (EC)DHE with PSK */
-    word16            sendAlertState:1;   /* nonblocking resume */
     word16            partialWrite:1;     /* only one msg per write call */
     word16            quietShutdown:1;    /* don't send close notify */
     word16            certOnly:1;         /* stop once we get cert */
@@ -3771,6 +3775,12 @@ typedef struct Options {
     word16            startedETMRead:1;       /* Doing Encrypt-Then-MAC read */
     word16            startedETMWrite:1;      /* Doing Encrypt-Then-MAC write */
 #endif
+#ifdef WOLFSSL_ASYNC_CRYPT
+    word16            buildArgsSet:1;         /* buildArgs are set and need to
+                                               * be free'd */
+#endif
+    word16            buildingMsg:1;      /* If set then we need to re-enter the
+                                           * handshake logic. */
 
     /* need full byte values for this section */
     byte            processReply;           /* nonblocking resume */
@@ -4212,17 +4222,16 @@ typedef struct BuildMsgArgs {
 } BuildMsgArgs;
 #endif
 
-#ifdef WOLFSSL_ASYNC_CRYPT
+#ifdef WOLFSSL_ASYNC_IO
     #define MAX_ASYNC_ARGS 18
     typedef void (*FreeArgsCb)(struct WOLFSSL* ssl, void* pArgs);
 
     struct WOLFSSL_ASYNC {
-        WC_ASYNC_DEV* dev;
-        FreeArgsCb    freeArgs; /* function pointer to cleanup args */
-        word32        args[MAX_ASYNC_ARGS]; /* holder for current args */
-#ifndef WOLFSSL_NO_TLS12
+#if defined(WOLFSSL_ASYNC_CRYPT) && !defined(WOLFSSL_NO_TLS12)
         BuildMsgArgs  buildArgs; /* holder for current BuildMessage args */
 #endif
+        FreeArgsCb    freeArgs; /* function pointer to cleanup args */
+        word32        args[MAX_ASYNC_ARGS]; /* holder for current args */
     };
 #endif
 
@@ -4300,10 +4309,13 @@ struct WOLFSSL {
     HandShakeDoneCb hsDoneCb;          /*  notify user handshake done */
     void*           hsDoneCtx;         /*  user handshake cb context  */
 #endif
+#ifdef WOLFSSL_ASYNC_IO
 #ifdef WOLFSSL_ASYNC_CRYPT
-    struct WOLFSSL_ASYNC async;
-#elif defined(WOLFSSL_NONBLOCK_OCSP)
-    void*           nonblockarg;        /* dynamic arg for handling non-block resume */
+    WC_ASYNC_DEV* asyncDev;
+#endif
+    /* Message building context should be stored here for functions that expect
+     * to encounter encryption blocking or fragment the message. */
+    struct WOLFSSL_ASYNC* async;
 #endif
     void*           hsKey;              /* Handshake key (RsaKey or ecc_key) allocated from heap */
     word32          hsType;             /* Type of Handshake key (hsKey) */
@@ -4319,6 +4331,7 @@ struct WOLFSSL {
     ClientSession*  clientSession;
 #endif
     WOLFSSL_ALERT_HISTORY alert_history;
+    WOLFSSL_ALERT   pendingAlert;
     int             error;
     int             rfd;                /* read  file descriptor */
     int             wfd;                /* write file descriptor */
@@ -4834,6 +4847,7 @@ WOLFSSL_LOCAL int SendServerKeyExchange(WOLFSSL* ssl);
 WOLFSSL_LOCAL int SendBuffered(WOLFSSL* ssl);
 WOLFSSL_LOCAL int ReceiveData(WOLFSSL* ssl, byte* output, int sz, int peek);
 WOLFSSL_LOCAL int SendFinished(WOLFSSL* ssl);
+WOLFSSL_LOCAL int RetrySendAlert(WOLFSSL* ssl);
 WOLFSSL_LOCAL int SendAlert(WOLFSSL* ssl, int severity, int type);
 WOLFSSL_LOCAL int ProcessReply(WOLFSSL* ssl);
 WOLFSSL_LOCAL int ProcessReplyEx(WOLFSSL* ssl, int allowSocketErr);

wolfssl/ssl.h

@@ -769,6 +769,7 @@ enum AlertDescription {
 
 
 enum AlertLevel {
+    alert_none = 0, /* Used to indicate no alert level is set */
     alert_warning = 1,
     alert_fatal   = 2
 };

wolfssl/wolfcrypt/settings.h

@@ -2707,6 +2707,16 @@ extern void uITRON4_free(void *p) ;
     #define NO_RC4
 #endif
 
+#if !defined(WOLFSSL_NO_ASYNC_IO) || defined(WOLFSSL_ASYNC_CRYPT) || \
+     defined(WOLFSSL_NONBLOCK_OCSP)
+    /* Enable asynchronous support in TLS functions to support one or more of
+     * the following:
+     * - re-entry after a network blocking return
+     * - re-entry after OCSP blocking return
+     * - asynchronous cryptography */
+    #undef WOLFSSL_ASYNC_IO
+    #define WOLFSSL_ASYNC_IO
+#endif
 
 #ifdef __cplusplus
     }   /* extern "C" */