urlapi: return CURLUE_BAD_HOSTNAME if puny2idn encoding fails

And document it. Only return out of memory when it actually is a memory
problem.

Pointed-out-by: Jacob Mealey
Closes #11674

docs/libcurl/curl_url_get.3

@@ -94,7 +94,8 @@ anything outside the ASCII range.
 .IP CURLU_PUNY2IDN
 If set and asked to retrieve the \fBCURLUPART_HOST\fP or \fBCURLUPART_URL\fP
 parts, libcurl returns the host name in its IDN (International Domain Name)
-UTF-8 version if it otherwise is a punycode version.
+UTF-8 version if it otherwise is a punycode version. If the punycode cannot be
+converted to IDN correct, libcurl returns \fICURLUE_BAD_HOSTNAME\fP.
 
 If libcurl is built without IDN capabilities, using this bit will make
 \fIcurl_url_get(3)\fP return \fICURLUE_LACKS_IDN\fP if the host name is using

lib/idn.c

@@ -68,11 +68,10 @@ WINBASEAPI int WINAPI IdnToUnicode(DWORD dwFlags,
 
 #define IDN_MAX_LENGTH 255
 
-bool Curl_win32_idn_to_ascii(const char *in, char **out)
+static CURLcode win32_idn_to_ascii(const char *in, char **out)
 {
-  bool success = FALSE;
-
   wchar_t *in_w = curlx_convert_UTF8_to_wchar(in);
+  *out = NULL;
   if(in_w) {
     wchar_t punycode[IDN_MAX_LENGTH];
     int chars = IdnToAscii(0, in_w, (int)(wcslen(in_w) + 1), punycode,
@@ -83,16 +82,20 @@ bool Curl_win32_idn_to_ascii(const char *in, char **out)
       if(mstr) {
         *out = strdup(mstr);
         curlx_unicodefree(mstr);
-        if(*out)
-          success = TRUE;
+        if(!*out)
+          return CURLE_OUT_OF_MEMORY;
       }
+      else
+        return CURLE_OUT_OF_MEMORY;
     }
+    else
+      return CURLE_URL_MALFORMAT;
   }
 
-  return success;
+  return CURLE_OK;
 }
 
-char *Curl_win32_ascii_to_idn(const char *in)
+static CURLcode win32_ascii_to_idn(const char *in, char **output)
 {
   char *out = NULL;
 
@@ -107,10 +110,17 @@ char *Curl_win32_ascii_to_idn(const char *in)
       if(mstr) {
         out = strdup(mstr);
         curlx_unicodefree(mstr);
+        if(!out)
+          return CURLE_OUT_OF_MEMORY;
       }
     }
+    else
+      return CURLE_URL_MALFORMAT;
   }
-  return out;
+  else
+    return CURLE_URL_MALFORMAT;
+  *output = out;
+  return CURLE_OK;
 }
 
 #endif /* USE_WIN32_IDN */
@@ -137,10 +147,15 @@ bool Curl_is_ASCII_name(const char *hostname)
 /*
  * Curl_idn_decode() returns an allocated IDN decoded string if it was
  * possible. NULL on error.
+ *
+ * CURLE_URL_MALFORMAT - the host name could not be converted
+ * CURLE_OUT_OF_MEMORY - memory problem
+ *
  */
-static char *idn_decode(const char *input)
+static CURLcode idn_decode(const char *input, char **output)
 {
   char *decoded = NULL;
+  CURLcode result = CURLE_OK;
 #ifdef USE_LIBIDN2
   if(idn2_check_version(IDN2_VERSION)) {
     int flags = IDN2_NFC_INPUT
@@ -157,52 +172,68 @@ static char *idn_decode(const char *input)
          compatibility */
       rc = IDN2_LOOKUP(input, &decoded, IDN2_TRANSITIONAL);
     if(rc != IDN2_OK)
-      decoded = NULL;
+      result = CURLE_URL_MALFORMAT;
   }
 #elif defined(USE_WIN32_IDN)
-  if(!Curl_win32_idn_to_ascii(input, &decoded))
-    decoded = NULL;
+  result = win32_idn_to_ascii(input, &decoded);
 #endif
-  return decoded;
+  if(!result)
+    *output = decoded;
+  return result;
 }
 
-static char *idn_encode(const char *puny)
+static CURLcode idn_encode(const char *puny, char **output)
 {
   char *enc = NULL;
 #ifdef USE_LIBIDN2
   int rc = idn2_to_unicode_8z8z(puny, &enc, 0);
   if(rc != IDNA_SUCCESS)
-    return NULL;
+    return rc == IDNA_MALLOC_ERROR ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT;
 #elif defined(USE_WIN32_IDN)
-  enc = Curl_win32_ascii_to_idn(puny);
+  CURLcode result = win32_ascii_to_idn(puny, &enc);
+  if(result)
+    return result;
 #endif
-  return enc;
+  *output = enc;
+  return CURLE_OK;
 }
 
-char *Curl_idn_decode(const char *input)
+CURLcode Curl_idn_decode(const char *input, char **output)
 {
-  char *d = idn_decode(input);
+  char *d = NULL;
+  CURLcode result = idn_decode(input, &d);
 #ifdef USE_LIBIDN2
-  if(d) {
+  if(!result) {
     char *c = strdup(d);
     idn2_free(d);
-    d = c;
+    if(c)
+      d = c;
+    else
+      result = CURLE_OUT_OF_MEMORY;
   }
 #endif
-  return d;
+  if(!result)
+    *output = d;
+  return result;
 }
 
-char *Curl_idn_encode(const char *puny)
+CURLcode Curl_idn_encode(const char *puny, char **output)
 {
-  char *d = idn_encode(puny);
+  char *d = NULL;
+  CURLcode result = idn_encode(puny, &d);
 #ifdef USE_LIBIDN2
-  if(d) {
+  if(!result) {
     char *c = strdup(d);
     idn2_free(d);
-    d = c;
+    if(c)
+      d = c;
+    else
+      result = CURLE_OUT_OF_MEMORY;
   }
 #endif
-  return d;
+  if(!result)
+    *output = d;
+  return result;
 }
 
 /*
@@ -230,8 +261,9 @@ CURLcode Curl_idnconvert_hostname(struct hostname *host)
 #ifdef USE_IDN
   /* Check name for non-ASCII and convert hostname if we can */
   if(!Curl_is_ASCII_name(host->name)) {
-    char *decoded = idn_decode(host->name);
-    if(decoded) {
+    char *decoded;
+    CURLcode result = idn_decode(host->name, &decoded);
+    if(!result) {
       if(!*decoded) {
         /* zero length is a bad host name */
         Curl_idn_free(decoded);
@@ -243,7 +275,7 @@ CURLcode Curl_idnconvert_hostname(struct hostname *host)
       host->name = host->encalloc;
     }
     else
-      return CURLE_URL_MALFORMAT;
+      return result;
   }
 #endif
   return CURLE_OK;

lib/idn.h

@@ -24,17 +24,13 @@
  *
  ***************************************************************************/
 
-#ifdef USE_WIN32_IDN
-bool Curl_win32_idn_to_ascii(const char *in, char **out);
-char *Curl_win32_ascii_to_idn(const char *in);
-#endif /* USE_WIN32_IDN */
 bool Curl_is_ASCII_name(const char *hostname);
 CURLcode Curl_idnconvert_hostname(struct hostname *host);
 #if defined(USE_LIBIDN2) || defined(USE_WIN32_IDN)
 #define USE_IDN
 void Curl_free_idnconverted_hostname(struct hostname *host);
-char *Curl_idn_decode(const char *input);
-char *Curl_idn_encode(const char *input);
+CURLcode Curl_idn_decode(const char *input, char **output);
+CURLcode Curl_idn_encode(const char *input, char **output);
 #ifdef USE_LIBIDN2
 #define Curl_idn_free(x) idn2_free(x)
 #else

lib/urlapi.c

@@ -1545,9 +1545,10 @@ CURLUcode curl_url_get(const CURLU *u, CURLUPart what,
 #ifndef USE_IDN
           return CURLUE_LACKS_IDN;
 #else
-          allochost = Curl_idn_decode(u->host);
-          if(!allochost)
-            return CURLUE_OUT_OF_MEMORY;
+          CURLcode result = Curl_idn_decode(u->host, &allochost);
+          if(result)
+            return (result == CURLE_OUT_OF_MEMORY) ?
+              CURLUE_OUT_OF_MEMORY : CURLUE_BAD_HOSTNAME;
 #endif
         }
       }
@@ -1556,9 +1557,11 @@ CURLUcode curl_url_get(const CURLU *u, CURLUPart what,
 #ifndef USE_IDN
           return CURLUE_LACKS_IDN;
 #else
-          allochost = Curl_idn_encode(u->host);
-          if(!allochost)
-            return CURLUE_OUT_OF_MEMORY;
+          CURLcode result = Curl_idn_encode(u->host, &allochost);
+          if(result)
+            /* this is the most likely error */
+            return (result == CURLE_OUT_OF_MEMORY) ?
+              CURLUE_OUT_OF_MEMORY : CURLUE_BAD_HOSTNAME;
 #endif
         }
       }
@@ -1632,9 +1635,11 @@ CURLUcode curl_url_get(const CURLU *u, CURLUPart what,
 #ifndef USE_IDN
         return CURLUE_LACKS_IDN;
 #else
-        char *allochost = Curl_idn_decode(*part);
-        if(!allochost)
-          return CURLUE_OUT_OF_MEMORY;
+        char *allochost;
+        CURLcode result = Curl_idn_decode(*part, &allochost);
+        if(result)
+          return (result == CURLE_OUT_OF_MEMORY) ?
+            CURLUE_OUT_OF_MEMORY : CURLUE_BAD_HOSTNAME;
         free(*part);
         *part = allochost;
 #endif
@@ -1645,9 +1650,11 @@ CURLUcode curl_url_get(const CURLU *u, CURLUPart what,
 #ifndef USE_IDN
         return CURLUE_LACKS_IDN;
 #else
-        char *allochost = Curl_idn_encode(*part);
-        if(!allochost)
-          return CURLUE_OUT_OF_MEMORY;
+        char *allochost;
+        CURLcode result = Curl_idn_encode(*part, &allochost);
+        if(result)
+          return (result == CURLE_OUT_OF_MEMORY) ?
+            CURLUE_OUT_OF_MEMORY : CURLUE_BAD_HOSTNAME;
         free(*part);
         *part = allochost;
 #endif