Merge pull request #1640 from SparkiDev/tls13_nb

Fix non-blocking and buffered I/O

examples/benchmark/tls_bench.c

@@ -59,9 +59,17 @@ bench_tls(args);
 #include <unistd.h>
 #include <sys/time.h>
 
+#if 0
+#define NON_BLOCKING
+#endif
+
 /* Defaults for configuration parameters */
 #define THREAD_PAIRS    1 /* Thread pairs of server/client */
-#define MEM_BUFFER_SZ   (1024*16) /* Must be large enough to handle max packet size */
+#ifndef NON_BLOCKING
+    #define MEM_BUFFER_SZ   (1024 * 16) /* Must be large enough to handle max packet size */
+#else
+    #define MEM_BUFFER_SZ   256
+#endif
 #define MIN_DHKEY_BITS  1024
 #define RUNTIME_SEC     1
 #define TEST_SIZE_BYTES (1024 * 1024)
@@ -230,11 +238,16 @@ static int ServerSend(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     pthread_mutex_lock(&info->to_client.mutex);
 
+#ifndef NON_BLOCKING
     /* check for overflow */
     if (info->to_client.write_idx + sz > MEM_BUFFER_SZ) {
         pthread_mutex_unlock(&info->to_client.mutex);
         return -1;
     }
+#else
+    if (info->to_client.write_idx + sz > MEM_BUFFER_SZ)
+        sz = MEM_BUFFER_SZ - info->to_client.write_idx;
+#endif
 
     memcpy(&info->to_client.buf[info->to_client.write_idx], buf, sz);
     info->to_client.write_idx += sz;
@@ -245,6 +258,10 @@ static int ServerSend(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     (void)ssl;
 
+#ifdef NON_BLOCKING
+    if (sz == 0)
+        return WOLFSSL_CBIO_ERR_WANT_WRITE;
+#endif
     return sz;
 }
 
@@ -256,8 +273,13 @@ static int ServerRecv(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     pthread_mutex_lock(&info->to_server.mutex);
 
+#ifndef NON_BLOCKING
     while (info->to_server.write_idx - info->to_server.read_idx < sz && !info->to_client.done)
         pthread_cond_wait(&info->to_server.cond, &info->to_server.mutex);
+#else
+    if (info->to_server.write_idx - info->to_server.read_idx < sz)
+        sz = info->to_server.write_idx - info->to_server.read_idx;
+#endif
 
     memcpy(buf, &info->to_server.buf[info->to_server.read_idx], sz);
     info->to_server.read_idx += sz;
@@ -276,6 +298,10 @@ static int ServerRecv(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     (void)ssl;
 
+#ifdef NON_BLOCKING
+    if (sz == 0)
+        return WOLFSSL_CBIO_ERR_WANT_READ;
+#endif
     return sz;
 }
 
@@ -287,11 +313,16 @@ static int ClientSend(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     pthread_mutex_lock(&info->to_server.mutex);
 
+#ifndef NON_BLOCKING
     /* check for overflow */
     if (info->to_client.write_idx + sz > MEM_BUFFER_SZ) {
         pthread_mutex_unlock(&info->to_server.mutex);
         return -1;
     }
+#else
+    if (info->to_server.write_idx + sz > MEM_BUFFER_SZ)
+        sz = MEM_BUFFER_SZ - info->to_server.write_idx;
+#endif
 
     memcpy(&info->to_server.buf[info->to_server.write_idx], buf, sz);
     info->to_server.write_idx += sz;
@@ -302,6 +333,10 @@ static int ClientSend(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     (void)ssl;
 
+#ifdef NON_BLOCKING
+    if (sz == 0)
+        return WOLFSSL_CBIO_ERR_WANT_WRITE;
+#endif
     return sz;
 }
 
@@ -313,8 +348,13 @@ static int ClientRecv(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     pthread_mutex_lock(&info->to_client.mutex);
 
+#ifndef NON_BLOCKING
     while (info->to_client.write_idx - info->to_client.read_idx < sz)
         pthread_cond_wait(&info->to_client.cond, &info->to_client.mutex);
+#else
+    if (info->to_client.write_idx - info->to_client.read_idx < sz)
+        sz = info->to_client.write_idx - info->to_client.read_idx;
+#endif
 
     memcpy(buf, &info->to_client.buf[info->to_client.read_idx], sz);
     info->to_client.read_idx += sz;
@@ -330,6 +370,10 @@ static int ClientRecv(WOLFSSL* ssl, char* buf, int sz, void* ctx)
 
     (void)ssl;
 
+#ifdef NON_BLOCKING
+    if (sz == 0)
+        return WOLFSSL_CBIO_ERR_WANT_READ;
+#endif
     return sz;
 }
 
@@ -350,12 +394,20 @@ static void* client_thread(void* args)
     unsigned char *writeBuf;
     double start;
     int ret, bufSize;
-    WOLFSSL_CTX* cli_ctx;
+    WOLFSSL_CTX* cli_ctx = NULL;
     WOLFSSL* cli_ssl;
     int haveShownPeerInfo = 0;
+    int tls13 = XSTRNCMP(info->cipher, "TLS13", 5) == 0;
 
     /* set up client */
-    cli_ctx = wolfSSL_CTX_new(wolfSSLv23_client_method());
+#ifdef WOLFSSL_TLS13
+    if (tls13)
+        cli_ctx = wolfSSL_CTX_new(wolfTLSv1_3_client_method());
+#endif
+#ifndef WOLFSSL_NO_TLS12
+    if (!tls13)
+        cli_ctx = wolfSSL_CTX_new(wolfTLSv1_2_client_method());
+#endif
     if (cli_ctx == NULL) err_sys("error creating ctx");
 
 #ifndef NO_CERTS
@@ -393,6 +445,9 @@ static void* client_thread(void* args)
     }
 
     while (!info->shutdown) {
+    #ifdef NON_BLOCKING
+        int err;
+    #endif
         cli_ssl = wolfSSL_new(cli_ctx);
         if (cli_ssl == NULL) err_sys("error creating client object");
 
@@ -401,7 +456,16 @@ static void* client_thread(void* args)
 
         /* perform connect */
         start = gettime_secs(1);
+    #ifndef NON_BLOCKING
         ret = wolfSSL_connect(cli_ssl);
+    #else
+        do
+        {
+            ret = wolfSSL_connect(cli_ssl);
+            err = wolfSSL_get_error(cli_ssl, ret);
+        }
+        while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE);
+    #endif
         start = gettime_secs(0) - start;
         if (ret != WOLFSSL_SUCCESS) {
             if (info->shutdown)
@@ -428,7 +492,16 @@ static void* client_thread(void* args)
             /* write test message to server */
             while (info->client_stats.rxTotal < info->numBytes) {
                 start = gettime_secs(1);
+            #ifndef NON_BLOCKING
                 ret = wolfSSL_write(cli_ssl, writeBuf, info->packetSize);
+            #else
+                do
+                {
+                    ret = wolfSSL_write(cli_ssl, writeBuf, info->packetSize);
+                    err = wolfSSL_get_error(cli_ssl, ret);
+                }
+                while (err == WOLFSSL_ERROR_WANT_WRITE);
+            #endif
                 info->client_stats.txTime += gettime_secs(0) - start;
                 if (ret > 0) {
                     info->client_stats.txTotal += ret;
@@ -436,7 +509,16 @@ static void* client_thread(void* args)
 
                 /* read echo of message */
                 start = gettime_secs(1);
+            #ifndef NON_BLOCKING
                 ret = wolfSSL_read(cli_ssl, buf, bufSize-1);
+            #else
+                do
+                {
+                    ret = wolfSSL_read(cli_ssl, buf, bufSize-1);
+                    err = wolfSSL_get_error(cli_ssl, ret);
+                }
+                while (err == WOLFSSL_ERROR_WANT_READ);
+            #endif
                 info->client_stats.rxTime += gettime_secs(0) - start;
                 if (ret > 0) {
                     info->client_stats.rxTotal += ret;
@@ -477,11 +559,19 @@ static void* server_thread(void* args)
     unsigned char *buf;
     double start;
     int ret, len = 0, bufSize;
-    WOLFSSL_CTX* srv_ctx;
+    WOLFSSL_CTX* srv_ctx = NULL;
     WOLFSSL* srv_ssl;
+    int tls13 = XSTRNCMP(info->cipher, "TLS13", 5) == 0;
 
     /* set up server */
-    srv_ctx = wolfSSL_CTX_new(wolfSSLv23_server_method());
+#ifdef WOLFSSL_TLS13
+    if (tls13)
+        srv_ctx = wolfSSL_CTX_new(wolfTLSv1_3_server_method());
+#endif
+#ifndef WOLFSSL_NO_TLS12
+    if (!tls13)
+        srv_ctx = wolfSSL_CTX_new(wolfTLSv1_2_server_method());
+#endif
     if (srv_ctx == NULL) err_sys("error creating server ctx");
 
 #ifndef NO_CERTS
@@ -521,6 +611,9 @@ static void* server_thread(void* args)
 #endif
 
     while (!info->shutdown) {
+    #ifdef NON_BLOCKING
+        int err;
+    #endif
         srv_ssl = wolfSSL_new(srv_ctx);
         if (srv_ssl == NULL) err_sys("error creating server object");
 
@@ -529,7 +622,16 @@ static void* server_thread(void* args)
 
         /* accept tls connection without tcp sockets */
         start = gettime_secs(1);
+    #ifndef NON_BLOCKING
         ret = wolfSSL_accept(srv_ssl);
+    #else
+        do
+        {
+            ret = wolfSSL_accept(srv_ssl);
+            err = wolfSSL_get_error(srv_ssl, ret);
+        }
+        while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE);
+    #endif
         start = gettime_secs(0) - start;
         if (ret != WOLFSSL_SUCCESS) {
             if (info->shutdown)
@@ -553,7 +655,16 @@ static void* server_thread(void* args)
                 /* read msg post handshake from client */
                 memset(buf, 0, bufSize);
                 start = gettime_secs(1);
+            #ifndef NON_BLOCKING
                 ret = wolfSSL_read(srv_ssl, buf, bufSize-1);
+            #else
+                do
+                {
+                    ret = wolfSSL_read(srv_ssl, buf, bufSize-1);
+                    err = wolfSSL_get_error(srv_ssl, ret);
+                }
+                while (err == WOLFSSL_ERROR_WANT_READ);
+            #endif
                 info->server_stats.rxTime += gettime_secs(0) - start;
                 if (ret > 0) {
                     info->server_stats.rxTotal += ret;
@@ -562,7 +673,16 @@ static void* server_thread(void* args)
 
                 /* write message back to client */
                 start = gettime_secs(1);
+            #ifndef NON_BLOCKING
                 ret = wolfSSL_write(srv_ssl, buf, len);
+            #else
+                do
+                {
+                    ret = wolfSSL_write(srv_ssl, buf, len);
+                    err = wolfSSL_get_error(srv_ssl, ret);
+                }
+                while (err == WOLFSSL_ERROR_WANT_WRITE);
+            #endif
                 info->server_stats.txTime += gettime_secs(0) - start;
                 if (ret > 0) {
                     info->server_stats.txTotal += ret;
@@ -608,7 +728,7 @@ static void print_stats(stats_t* wcStat, const char* desc, const char* cipher, i
                "\tConnect Avg : %9.3f ms\n";
     }
     else {
-        formatStr = "%s\t%s\t%d\t%9d\t%9.3f\t%9.3f\t%9.3f\t%9.3f\t%9.3f\t%9.3f\n";
+        formatStr = "%-6s  %-33s  %11d  %9d  %9.3f  %9.3f  %9.3f  %9.3f  %17.3f  %15.3f\n";
     }
 
     printf(formatStr,
@@ -847,7 +967,9 @@ int bench_tls(void* args)
             printf("Totals for %d Threads\n", argThreadPairs);
         }
         else {
-            printf("Side\tCipher\tTotal Bytes\tNum Conns\tRx ms\tTx ms\tRx MB/s\tTx MB/s\tConnect Total ms\tConnect Avg ms\n");
+            printf("%-6s  %-33s  %11s  %9s  %9s  %9s  %9s  %9s  %17s  %15s\n",
+                "Side", "Cipher", "Total Bytes", "Num Conns", "Rx ms", "Tx ms",
+                "Rx MB/s", "Tx MB/s", "Connect Total ms", "Connect Avg ms");
             print_stats(&srv_comb, "Server", theadInfo[0].cipher, 0);
             print_stats(&cli_comb, "Client", theadInfo[0].cipher, 0);
         }

src/tls13.c

@@ -5995,9 +5995,6 @@ static int SendTls13Finished(WOLFSSL* ssl)
 
     ssl->buffers.outputBuffer.length += sendSz;
 
-    if ((ret = SendBuffered(ssl)) != 0)
-        return ret;
-
     if (ssl->options.side == WOLFSSL_SERVER_END) {
         /* Can send application data now. */
         if ((ret = DeriveMasterSecret(ssl)) != 0)
@@ -6037,6 +6034,8 @@ static int SendTls13Finished(WOLFSSL* ssl)
 
 #if defined(HAVE_SESSION_TICKET) || !defined(NO_PSK)
         ret = DeriveResumptionSecret(ssl, ssl->session.masterSecret);
+        if (ret != 0)
+            return ret;
 #endif
     }
 
@@ -6053,6 +6052,9 @@ static int SendTls13Finished(WOLFSSL* ssl)
     }
 #endif
 
+    if ((ret = SendBuffered(ssl)) != 0)
+        return ret;
+
     WOLFSSL_LEAVE("SendTls13Finished", ret);
     WOLFSSL_END(WC_FUNC_FINISHED_SEND);
 
@@ -7470,6 +7472,12 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
                 ssl->options.sentChangeCipher = 1;
             }
         #endif
+
+            ssl->options.connectState = FIRST_REPLY_SECOND;
+            WOLFSSL_MSG("connect state: FIRST_REPLY_SECOND");
+            FALL_THROUGH;
+
+        case FIRST_REPLY_SECOND:
         #ifndef NO_CERTS
             if (!ssl->options.resuming && ssl->options.sendVerify) {
                 ssl->error = SendTls13Certificate(ssl);
@@ -7481,12 +7489,11 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
             }
         #endif
 
-            ssl->options.connectState = FIRST_REPLY_SECOND;
-            WOLFSSL_MSG("connect state: FIRST_REPLY_SECOND");
+            ssl->options.connectState = FIRST_REPLY_THIRD;
+            WOLFSSL_MSG("connect state: FIRST_REPLY_THIRD");
             FALL_THROUGH;
 
-        case FIRST_REPLY_SECOND:
-
+        case FIRST_REPLY_THIRD:
         #ifndef NO_CERTS
             if (!ssl->options.resuming && ssl->options.sendVerify) {
                 ssl->error = SendTls13CertificateVerify(ssl);
@@ -7498,11 +7505,11 @@ int wolfSSL_connect_TLSv13(WOLFSSL* ssl)
             }
         #endif
 
-            ssl->options.connectState = FIRST_REPLY_THIRD;
-            WOLFSSL_MSG("connect state: FIRST_REPLY_THIRD");
+            ssl->options.connectState = FIRST_REPLY_FOURTH;
+            WOLFSSL_MSG("connect state: FIRST_REPLY_FOURTH");
             FALL_THROUGH;
 
-        case FIRST_REPLY_THIRD:
+        case FIRST_REPLY_FOURTH:
             if ((ssl->error = SendTls13Finished(ssl)) != 0) {
                 WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
@@ -7989,7 +7996,7 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
 
     switch (ssl->options.acceptState) {
 
-        case ACCEPT_BEGIN :
+        case TLS13_ACCEPT_BEGIN :
             /* get client_hello */
             while (ssl->options.clientState < CLIENT_HELLO_COMPLETE) {
                 if ((ssl->error = ProcessReply(ssl)) < 0) {
@@ -7998,11 +8005,11 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 }
             }
 
-            ssl->options.acceptState = ACCEPT_CLIENT_HELLO_DONE;
+            ssl->options.acceptState = TLS13_ACCEPT_CLIENT_HELLO_DONE;
             WOLFSSL_MSG("accept state ACCEPT_CLIENT_HELLO_DONE");
             FALL_THROUGH;
 
-        case ACCEPT_CLIENT_HELLO_DONE :
+        case TLS13_ACCEPT_CLIENT_HELLO_DONE :
 #ifdef WOLFSSL_TLS13_DRAFT_18
             if (ssl->options.serverState ==
                                           SERVER_HELLO_RETRY_REQUEST_COMPLETE) {
@@ -8011,6 +8018,12 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                     return WOLFSSL_FATAL_ERROR;
                 }
             }
+
+            ssl->options.acceptState = TLS13_ACCEPT_FIRST_REPLY_DONE;
+            WOLFSSL_MSG("accept state ACCEPT_FIRST_REPLY_DONE");
+            FALL_THROUGH;
+
+        case TLS13_ACCEPT_HELLO_RETRY_REQUEST_DONE :
 #else
             if (ssl->options.serverState ==
                                           SERVER_HELLO_RETRY_REQUEST_COMPLETE) {
@@ -8019,20 +8032,29 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                     WOLFSSL_ERROR(ssl->error);
                     return WOLFSSL_FATAL_ERROR;
                 }
+            }
+            
+            ssl->options.acceptState = TLS13_ACCEPT_HELLO_RETRY_REQUEST_DONE;
+            WOLFSSL_MSG("accept state ACCEPT_HELLO_RETRY_REQUEST_DONE");
+            FALL_THROUGH;
+
+        case TLS13_ACCEPT_HELLO_RETRY_REQUEST_DONE :
     #ifdef WOLFSSL_TLS13_MIDDLEBOX_COMPAT
+            if (ssl->options.serverState ==
+                                          SERVER_HELLO_RETRY_REQUEST_COMPLETE) {
                 if ((ssl->error = SendChangeCipher(ssl)) != 0) {
                     WOLFSSL_ERROR(ssl->error);
                     return WOLFSSL_FATAL_ERROR;
                 }
                 ssl->options.sentChangeCipher = 1;
-    #endif
             }
-#endif
-            ssl->options.acceptState = ACCEPT_HELLO_RETRY_REQUEST_DONE;
-            WOLFSSL_MSG("accept state ACCEPT_HELLO_RETRY_REQUEST_DONE");
+    #endif
+            ssl->options.acceptState = TLS13_ACCEPT_FIRST_REPLY_DONE;
+            WOLFSSL_MSG("accept state ACCEPT_FIRST_REPLY_DONE");
             FALL_THROUGH;
+#endif
 
-        case ACCEPT_HELLO_RETRY_REQUEST_DONE :
+        case TLS13_ACCEPT_FIRST_REPLY_DONE :
             if (ssl->options.serverState ==
                                           SERVER_HELLO_RETRY_REQUEST_COMPLETE) {
                 ssl->options.clientState = NULL_STATE;
@@ -8043,15 +8065,21 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                     }
                 }
             }
-            ssl->options.acceptState = ACCEPT_FIRST_REPLY_DONE;
-            WOLFSSL_MSG("accept state ACCEPT_FIRST_REPLY_DONE");
+
+            ssl->options.acceptState = TLS13_ACCEPT_SECOND_REPLY_DONE;
+            WOLFSSL_MSG("accept state ACCEPT_SECOND_REPLY_DONE");
             FALL_THROUGH;
 
-        case ACCEPT_FIRST_REPLY_DONE :
+        case TLS13_ACCEPT_SECOND_REPLY_DONE :
             if ((ssl->error = SendTls13ServerHello(ssl, server_hello)) != 0) {
                 WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
+            ssl->options.acceptState = TLS13_SERVER_HELLO_SENT;
+            WOLFSSL_MSG("accept state SERVER_HELLO_SENT");
+            FALL_THROUGH;
+
+        case TLS13_SERVER_HELLO_SENT :
     #if !defined(WOLFSSL_TLS13_DRAFT_18) && \
                                          defined(WOLFSSL_TLS13_MIDDLEBOX_COMPAT)
             if (!ssl->options.sentChangeCipher) {
@@ -8062,27 +8090,27 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 ssl->options.sentChangeCipher = 1;
             }
     #endif
+            
+            ssl->options.acceptState = TLS13_ACCEPT_THIRD_REPLY_DONE;
+            WOLFSSL_MSG("accept state ACCEPT_THIRD_REPLY_DONE");
+            FALL_THROUGH;
 
+        case TLS13_ACCEPT_THIRD_REPLY_DONE :
             if (!ssl->options.noPskDheKe) {
                 ssl->error = TLSX_KeyShare_DeriveSecret(ssl);
                 if (ssl->error != 0)
                     return WOLFSSL_FATAL_ERROR;
             }
 
-            ssl->options.acceptState = SERVER_HELLO_SENT;
-            WOLFSSL_MSG("accept state SERVER_HELLO_SENT");
-            FALL_THROUGH;
-
-        case SERVER_HELLO_SENT :
             if ((ssl->error = SendTls13EncryptedExtensions(ssl)) != 0) {
                 WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
-            ssl->options.acceptState = SERVER_EXTENSIONS_SENT;
+            ssl->options.acceptState = TLS13_SERVER_EXTENSIONS_SENT;
             WOLFSSL_MSG("accept state SERVER_EXTENSIONS_SENT");
             FALL_THROUGH;
 
-        case SERVER_EXTENSIONS_SENT :
+        case TLS13_SERVER_EXTENSIONS_SENT :
 #ifndef NO_CERTS
             if (!ssl->options.resuming) {
                 if (ssl->options.verifyPeer) {
@@ -8094,12 +8122,11 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 }
             }
 #endif
-            ssl->options.acceptState = CERT_REQ_SENT;
+            ssl->options.acceptState = TLS13_CERT_REQ_SENT;
             WOLFSSL_MSG("accept state CERT_REQ_SENT");
             FALL_THROUGH;
 
-        case CERT_REQ_SENT :
-            ssl->options.acceptState = KEY_EXCHANGE_SENT;
+        case TLS13_CERT_REQ_SENT :
 #ifndef NO_CERTS
             if (!ssl->options.resuming && ssl->options.sendVerify) {
                 if ((ssl->error = SendTls13Certificate(ssl)) != 0) {
@@ -8108,11 +8135,11 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 }
             }
 #endif
-            ssl->options.acceptState = CERT_SENT;
+            ssl->options.acceptState = TLS13_CERT_SENT;
             WOLFSSL_MSG("accept state CERT_SENT");
             FALL_THROUGH;
 
-        case CERT_SENT :
+        case TLS13_CERT_SENT :
 #ifndef NO_CERTS
             if (!ssl->options.resuming && ssl->options.sendVerify) {
                 if ((ssl->error = SendTls13CertificateVerify(ssl)) != 0) {
@@ -8121,18 +8148,18 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 }
             }
 #endif
-            ssl->options.acceptState = CERT_STATUS_SENT;
-            WOLFSSL_MSG("accept state CERT_STATUS_SENT");
+            ssl->options.acceptState = TLS13_CERT_VERIFY_SENT;
+            WOLFSSL_MSG("accept state CERT_VERIFY_SENT");
             FALL_THROUGH;
 
-        case CERT_VERIFY_SENT :
+        case TLS13_CERT_VERIFY_SENT :
             if ((ssl->error = SendTls13Finished(ssl)) != 0) {
                 WOLFSSL_ERROR(ssl->error);
                 return WOLFSSL_FATAL_ERROR;
             }
 
-            ssl->options.acceptState = ACCEPT_FINISHED_DONE;
-            WOLFSSL_MSG("accept state ACCEPT_FINISHED_DONE");
+            ssl->options.acceptState = TLS13_ACCEPT_FINISHED_SENT;
+            WOLFSSL_MSG("accept state ACCEPT_FINISHED_SENT");
 #ifdef WOLFSSL_EARLY_DATA
             if (ssl->earlyData != no_early_data) {
                 ssl->options.handShakeState = SERVER_FINISHED_COMPLETE;
@@ -8141,7 +8168,7 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
 #endif
             FALL_THROUGH;
 
-        case ACCEPT_FINISHED_DONE :
+        case TLS13_ACCEPT_FINISHED_SENT :
 #ifdef HAVE_SESSION_TICKET
     #ifdef WOLFSSL_TLS13_TICKET_BEFORE_FINISHED
             if (!ssl->options.resuming && !ssl->options.verifyPeer &&
@@ -8153,22 +8180,22 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
             }
     #endif
 #endif /* HAVE_SESSION_TICKET */
-            ssl->options.acceptState = TICKET_SENT;
+            ssl->options.acceptState = TLS13_PRE_TICKET_SENT;
             WOLFSSL_MSG("accept state  TICKET_SENT");
             FALL_THROUGH;
 
-        case TICKET_SENT:
+        case TLS13_PRE_TICKET_SENT :
             while (ssl->options.clientState < CLIENT_FINISHED_COMPLETE)
                 if ( (ssl->error = ProcessReply(ssl)) < 0) {
                     WOLFSSL_ERROR(ssl->error);
                     return WOLFSSL_FATAL_ERROR;
                 }
 
-            ssl->options.acceptState = ACCEPT_SECOND_REPLY_DONE;
-            WOLFSSL_MSG("accept state ACCEPT_SECOND_REPLY_DONE");
+            ssl->options.acceptState = TLS13_ACCEPT_FINISHED_DONE;
+            WOLFSSL_MSG("accept state ACCEPT_FINISHED_DONE");
             FALL_THROUGH;
 
-        case ACCEPT_SECOND_REPLY_DONE :
+        case TLS13_ACCEPT_FINISHED_DONE :
 #ifdef HAVE_SESSION_TICKET
     #ifdef WOLFSSL_TLS13_TICKET_BEFORE_FINISHED
             if (!ssl->options.verifyPeer) {
@@ -8183,11 +8210,11 @@ int wolfSSL_accept_TLSv13(WOLFSSL* ssl)
                 }
             }
 #endif /* HAVE_SESSION_TICKET */
-            ssl->options.acceptState = ACCEPT_THIRD_REPLY_DONE;
-            WOLFSSL_MSG("accept state ACCEPT_THIRD_REPLY_DONE");
+            ssl->options.acceptState = TLS13_TICKET_SENT;
+            WOLFSSL_MSG("accept state TICKET_SENT");
             FALL_THROUGH;
 
-        case ACCEPT_THIRD_REPLY_DONE:
+        case TLS13_TICKET_SENT :
 #ifndef NO_HANDSHAKE_DONE_CB
             if (ssl->hsDoneCb) {
                 int cbret = ssl->hsDoneCb(ssl, ssl->hsDoneCtx);

wolfssl/internal.h

@@ -2904,6 +2904,25 @@ enum AcceptState {
     ACCEPT_THIRD_REPLY_DONE
 };
 
+/* TLS 1.3 server accept state for nonblocking restart */
+enum AcceptStateTls13 {
+    TLS13_ACCEPT_BEGIN = 0,
+    TLS13_ACCEPT_CLIENT_HELLO_DONE,
+    TLS13_ACCEPT_HELLO_RETRY_REQUEST_DONE,
+    TLS13_ACCEPT_FIRST_REPLY_DONE,
+    TLS13_ACCEPT_SECOND_REPLY_DONE,
+    TLS13_SERVER_HELLO_SENT,
+    TLS13_ACCEPT_THIRD_REPLY_DONE,
+    TLS13_SERVER_EXTENSIONS_SENT,
+    TLS13_CERT_REQ_SENT,
+    TLS13_CERT_SENT,
+    TLS13_CERT_VERIFY_SENT,
+    TLS13_ACCEPT_FINISHED_SENT,
+    TLS13_PRE_TICKET_SENT,
+    TLS13_ACCEPT_FINISHED_DONE,
+    TLS13_TICKET_SENT
+};
+
 /* buffers for struct WOLFSSL */
 typedef struct Buffers {
     bufferStatic    inputBuffer;