Merge pull request #3933 from miyazakh/rand_bytes_regression

fix retrun code regression on RAND_bytes

src/ssl.c

@@ -32321,7 +32321,11 @@ int wolfSSL_RAND_bytes(unsigned char* buf, int num)
 #endif
 
     WOLFSSL_ENTER("wolfSSL_RAND_bytes");
-
+    /* sanity check */
+    if (buf == NULL || num < 0)
+        /* return code compliant with OpenSSL */
+        return 0;
+        
     /* if a RAND callback has been set try and use it */
 #ifndef WOLFSSL_NO_OPENSSL_RAND_CB
     if (wolfSSL_RAND_InitMutex() == 0 && wc_LockMutex(&gRandMethodMutex) == 0) {

tests/api.c

@@ -31022,6 +31022,9 @@ static void test_wolfSSL_RAND_set_rand_method(void)
 
     printf(testingFmt, "wolfSSL_RAND_set_rand_method()");
 
+    buf = (byte*)XMALLOC(32 * sizeof(byte), NULL,
+                                               DYNAMIC_TYPE_TMP_BUFFER);
+                                                     
     AssertIntNE(wolfSSL_RAND_status(), 5432);
     AssertIntEQ(*was_cleanup_called, 0);
     wolfSSL_RAND_Cleanup();
@@ -31058,6 +31061,8 @@ static void test_wolfSSL_RAND_set_rand_method(void)
     wolfSSL_RAND_Cleanup();
     AssertIntEQ(*was_cleanup_called, 0);
 
+    XFREE(buf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    
     printf(resultFmt, passed);
 #endif /* OPENSSL_EXTRA && !WOLFSSL_NO_OPENSSL_RAND_CB */
 }
@@ -31073,17 +31078,24 @@ static void test_wolfSSL_RAND_bytes(void)
     byte *my_buf;
 
     printf(testingFmt, "test_wolfSSL_RAND_bytes()");
-
+    /* sanity check */
+    AssertIntEQ(RAND_bytes(NULL, 16), 0);
+    AssertIntEQ(RAND_bytes(NULL, 0), 0);
+    
     max_bufsize = size4;
 
     my_buf = (byte*)XMALLOC(max_bufsize * sizeof(byte), NULL,
                                                      DYNAMIC_TYPE_TMP_BUFFER);
+    
+    AssertIntEQ(RAND_bytes(my_buf, 0), 1);
+    AssertIntEQ(RAND_bytes(my_buf, -1), 0);
+    
     AssertNotNull(my_buf);
     XMEMSET(my_buf, 0, max_bufsize);
-    AssertIntEQ(wolfSSL_RAND_bytes(my_buf, size1), 1);
-    AssertIntEQ(wolfSSL_RAND_bytes(my_buf, size2), 1);
-    AssertIntEQ(wolfSSL_RAND_bytes(my_buf, size3), 1);
-    AssertIntEQ(wolfSSL_RAND_bytes(my_buf, size4), 1);
+    AssertIntEQ(RAND_bytes(my_buf, size1), 1);
+    AssertIntEQ(RAND_bytes(my_buf, size2), 1);
+    AssertIntEQ(RAND_bytes(my_buf, size3), 1);
+    AssertIntEQ(RAND_bytes(my_buf, size4), 1);
 
     XFREE(my_buf, NULL, DYNAMIC_TYPE_TMP_BUFFER);