vtls: fix tls proxy peer verification

- When verifying a proxy certificate for an ip address, use the correct
  ip family.

Prior to this change the "connection" ip family was used, which was not
necessarily the same.

Reported-by: HsiehYuho@users.noreply.github.com

Fixes https://github.com/curl/curl/issues/12831
Closes https://github.com/curl/curl/pull/12931

lib/urldata.h

@@ -266,11 +266,17 @@ typedef enum {
 /* SSL backend-specific data; declared differently by each SSL backend */
 struct ssl_backend_data;
 
+typedef enum {
+  CURL_SSL_PEER_DNS,
+  CURL_SSL_PEER_IPV4,
+  CURL_SSL_PEER_IPV6
+} ssl_peer_type;
+
 struct ssl_peer {
   char *hostname;        /* hostname for verification */
   char *dispname;        /* display version of hostname */
   char *sni;             /* SNI version of hostname or NULL if not usable */
-  BIT(is_ip_address);    /* if hostname is an IPv4|6 address */
+  ssl_peer_type type;    /* type of the peer information */
 };
 
 struct ssl_primary_config {

lib/vtls/bearssl.c

@@ -707,7 +707,7 @@ static CURLcode bearssl_connect_step1(struct Curl_cfilter *cf,
     infof(data, VTLS_INFOF_ALPN_OFFER_1STR, proto.data);
   }
 
-  if(connssl->peer.is_ip_address) {
+  if(connssl->peer.type != CURL_SSL_PEER_DNS) {
     if(verifyhost) {
       failf(data, "BearSSL: "
             "host verification of IP address is not supported");

lib/vtls/openssl.c

@@ -2134,7 +2134,7 @@ CURLcode Curl_ossl_verifyhost(struct Curl_easy *data, struct connectdata *conn,
                               struct ssl_peer *peer, X509 *server_cert)
 {
   bool matched = FALSE;
-  int target = GEN_DNS; /* target type, GEN_DNS or GEN_IPADD */
+  int target; /* target type, GEN_DNS or GEN_IPADD */
   size_t addrlen = 0;
   STACK_OF(GENERAL_NAME) *altnames;
 #ifdef ENABLE_IPV6
@@ -2149,19 +2149,28 @@ CURLcode Curl_ossl_verifyhost(struct Curl_easy *data, struct connectdata *conn,
 
   (void)conn;
   hostlen = strlen(peer->hostname);
-  if(peer->is_ip_address) {
+  switch(peer->type) {
+  case CURL_SSL_PEER_IPV4:
+    if(!Curl_inet_pton(AF_INET, peer->hostname, &addr))
+      return CURLE_PEER_FAILED_VERIFICATION;
+    target = GEN_IPADD;
+    addrlen = sizeof(struct in_addr);
+    break;
 #ifdef ENABLE_IPV6
-    if(conn->bits.ipv6_ip &&
-       Curl_inet_pton(AF_INET6, peer->hostname, &addr)) {
-      target = GEN_IPADD;
-      addrlen = sizeof(struct in6_addr);
-    }
-    else
+  case CURL_SSL_PEER_IPV6:
+    if(!Curl_inet_pton(AF_INET6, peer->hostname, &addr))
+      return CURLE_PEER_FAILED_VERIFICATION;
+    target = GEN_IPADD;
+    addrlen = sizeof(struct in6_addr);
+    break;
 #endif
-      if(Curl_inet_pton(AF_INET, peer->hostname, &addr)) {
-        target = GEN_IPADD;
-        addrlen = sizeof(struct in_addr);
-      }
+  case CURL_SSL_PEER_DNS:
+    target = GEN_DNS;
+    break;
+  default:
+    DEBUGASSERT(0);
+    failf(data, "unexpected ssl peer type: %d", peer->type);
+    return CURLE_PEER_FAILED_VERIFICATION;
   }
 
   /* get a "list" of alternative names */
@@ -2242,11 +2251,12 @@ CURLcode Curl_ossl_verifyhost(struct Curl_easy *data, struct connectdata *conn,
     /* an alternative name matched */
     ;
   else if(dNSName || iPAddress) {
-    infof(data, " subjectAltName does not match %s %s",
-          peer->is_ip_address? "ip address" : "host name", peer->dispname);
+    const char *tname = (peer->type == CURL_SSL_PEER_DNS) ? "host name" :
+                        (peer->type == CURL_SSL_PEER_IPV4) ?
+                        "ipv4 address" : "ipv6 address";
+    infof(data, " subjectAltName does not match %s %s", tname, peer->dispname);
     failf(data, "SSL: no alternative certificate subject name matches "
-          "target %s '%s'",
-          peer->is_ip_address? "ip address" : "host name", peer->dispname);
+          "target %s '%s'", tname, peer->dispname);
     result = CURLE_PEER_FAILED_VERIFICATION;
   }
   else {

lib/vtls/schannel.c

@@ -1159,7 +1159,7 @@ schannel_connect_step1(struct Curl_cfilter *cf, struct Curl_easy *data)
   }
 
   /* Warn if SNI is disabled due to use of an IP address */
-  if(connssl->peer.is_ip_address) {
+  if(connssl->peer.type != CURL_SSL_PEER_DNS) {
     infof(data, "schannel: using IP address, SNI is not supported by OS.");
   }
 

lib/vtls/sectransp.c

@@ -2008,7 +2008,7 @@ static CURLcode sectransp_connect_step1(struct Curl_cfilter *cf,
       return CURLE_SSL_CONNECT_ERROR;
     }
 
-    if(connssl->peer.is_ip_address) {
+    if(connssl->peer.type != CURL_SSL_PEER_DNS) {
       infof(data, "WARNING: using IP address, SNI is being disabled by "
             "the OS.");
     }

lib/vtls/vtls.c

@@ -1516,7 +1516,7 @@ void Curl_ssl_peer_cleanup(struct ssl_peer *peer)
   free(peer->sni);
   free(peer->hostname);
   peer->hostname = peer->sni = peer->dispname = NULL;
-  peer->is_ip_address = FALSE;
+  peer->type = CURL_SSL_PEER_DNS;
 }
 
 static void cf_close(struct Curl_cfilter *cf, struct Curl_easy *data)
@@ -1530,18 +1530,23 @@ static void cf_close(struct Curl_cfilter *cf, struct Curl_easy *data)
   cf->connected = FALSE;
 }
 
-static int is_ip_address(const char *hostname)
+static ssl_peer_type get_peer_type(const char *hostname)
 {
+  if(hostname && hostname[0]) {
 #ifdef ENABLE_IPV6
-  struct in6_addr addr;
+    struct in6_addr addr;
 #else
-  struct in_addr addr;
+    struct in_addr addr;
 #endif
-  return (hostname && hostname[0] && (Curl_inet_pton(AF_INET, hostname, &addr)
+    if(Curl_inet_pton(AF_INET, hostname, &addr))
+      return CURL_SSL_PEER_IPV4;
 #ifdef ENABLE_IPV6
-          || Curl_inet_pton(AF_INET6, hostname, &addr)
+    else if(Curl_inet_pton(AF_INET6, hostname, &addr)) {
+      return CURL_SSL_PEER_IPV6;
+    }
 #endif
-         ));
+  }
+  return CURL_SSL_PEER_DNS;
 }
 
 CURLcode Curl_ssl_peer_init(struct ssl_peer *peer, struct Curl_cfilter *cf)
@@ -1570,6 +1575,7 @@ CURLcode Curl_ssl_peer_init(struct ssl_peer *peer, struct Curl_cfilter *cf)
   }
 
   /* change if ehostname changed */
+  DEBUGASSERT(!ehostname || ehostname[0]);
   if(ehostname && (!peer->hostname
                    || strcmp(ehostname, peer->hostname))) {
     Curl_ssl_peer_cleanup(peer);
@@ -1589,8 +1595,8 @@ CURLcode Curl_ssl_peer_init(struct ssl_peer *peer, struct Curl_cfilter *cf)
     }
 
     peer->sni = NULL;
-    peer->is_ip_address = is_ip_address(peer->hostname)? TRUE : FALSE;
-    if(peer->hostname[0] && !peer->is_ip_address) {
+    peer->type = get_peer_type(peer->hostname);
+    if(peer->type == CURL_SSL_PEER_DNS && peer->hostname[0]) {
       /* not an IP address, normalize according to RCC 6066 ch. 3,
        * max len of SNI is 2^16-1, no trailing dot */
       size_t len = strlen(peer->hostname);