noproxy: support proxies specified using cidr notation

For both IPv4 and IPv6 addresses. Now also checks IPv6 addresses "correctly"
and not with string comparisons.

Split out the noproxy checks and functionality into noproxy.c

Added unit test 1614 to verify checking functions.

Reported-by: Mathieu Carbonneaux

Fixes #9773
Fixes #5745
Closes #9775

docs/KNOWN_BUGS

@@ -119,7 +119,6 @@ problems may have been fixed or changed somewhat since this was written.
  11.9 DoH does not inherit all transfer options
  11.10 Blocking socket operations in non-blocking API
  11.11 A shared connection cache is not thread-safe
- 11.12 'no_proxy' string-matches IPv6 numerical addresses
  11.14 Multi perform hangs waiting for threaded resolver
  11.15 CURLOPT_OPENSOCKETPAIRFUNCTION is missing
  11.16 libcurl uses renames instead of locking for atomic operations
@@ -931,13 +930,6 @@ problems may have been fixed or changed somewhat since this was written.
 
  See https://github.com/curl/curl/issues/4915 and lib1541.c
 
-11.12 'no_proxy' string-matches IPv6 numerical addresses
-
- This has the downside that "::1" for example does not match "::0:1" even
- though they are in fact the same address.
-
- See https://github.com/curl/curl/issues/5745
-
 11.14 Multi perform hangs waiting for threaded resolver
 
  If a threaded resolver takes a long time to complete, libcurl can be blocked

docs/libcurl/opts/CURLOPT_NOPROXY.3

@@ -53,10 +53,6 @@ brackets:
 
  "example.com,::1,localhost"
 
-IPv6 numerical addresses are compared as strings, so they will only match if
-the representations are the same: "::1" is the same as "::0:1" but they do not
-match.
-
 The application does not have to keep the string around after setting this
 option.
 .SH "Environment variables"

lib/Makefile.inc

@@ -177,6 +177,7 @@ LIB_CFILES =         \
   multi.c            \
   netrc.c            \
   nonblock.c         \
+  noproxy.c          \
   openldap.c         \
   parsedate.c        \
   pingpong.c         \
@@ -301,6 +302,7 @@ LIB_HFILES =         \
   multiif.h          \
   netrc.h            \
   nonblock.h         \
+  noproxy.h          \
   parsedate.h        \
   pingpong.h         \
   pop3.h             \

lib/noproxy.c

@@ -0,0 +1,212 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * SPDX-License-Identifier: curl
+ *
+ ***************************************************************************/
+
+#include "curl_setup.h"
+
+#ifndef CURL_DISABLE_PROXY
+
+#include "inet_pton.h"
+#include "strcase.h"
+#include "noproxy.h"
+
+/*
+ * Curl_cidr4_match() returns TRUE if the given IPv4 address is within the
+ * specified CIDR address range.
+ */
+UNITTEST bool Curl_cidr4_match(const char *ipv4,    /* 1.2.3.4 address */
+                               const char *network, /* 1.2.3.4 address */
+                               unsigned int bits)
+{
+  unsigned int address = 0;
+  unsigned int check = 0;
+
+  if(bits > 32)
+    /* strange input */
+    return FALSE;
+
+  if(1 != Curl_inet_pton(AF_INET, ipv4, &address))
+    return FALSE;
+  if(1 != Curl_inet_pton(AF_INET, network, &check))
+    return FALSE;
+
+  if(bits && (bits != 32)) {
+    unsigned int mask = 0xffffffff << (32 - bits);
+    unsigned int haddr = htonl(address);
+    unsigned int hcheck = htonl(check);
+#if 0
+    fprintf(stderr, "Host %s (%x) network %s (%x) bits %u mask %x => %x\n",
+            ipv4, haddr, network, hcheck, bits, mask,
+            (haddr ^ hcheck) & mask);
+#endif
+    if((haddr ^ hcheck) & mask)
+      return FALSE;
+    return TRUE;
+  }
+  return (address == check);
+}
+
+UNITTEST bool Curl_cidr6_match(const char *ipv6,
+                               const char *network,
+                               unsigned int bits)
+{
+  int bytes;
+  int rest;
+  unsigned char address[16];
+  unsigned char check[16];
+
+  if(!bits)
+    bits = 128;
+
+  bytes = bits/8;
+  rest = bits & 0x07;
+  if(1 != Curl_inet_pton(AF_INET6, ipv6, address))
+    return FALSE;
+  if(1 != Curl_inet_pton(AF_INET6, network, check))
+    return FALSE;
+  if((bytes > 16) || ((bytes == 16) && rest))
+    return FALSE;
+  if(bytes && memcmp(address, check, bytes))
+    return FALSE;
+  if(rest && !((address[bytes] ^ check[bytes]) & (0xff << (8 - rest))))
+    return FALSE;
+
+  return TRUE;
+}
+
+enum nametype {
+  TYPE_HOST,
+  TYPE_IPV4,
+  TYPE_IPV6
+};
+
+/****************************************************************
+* Checks if the host is in the noproxy list. returns TRUE if it matches and
+* therefore the proxy should NOT be used.
+****************************************************************/
+bool Curl_check_noproxy(const char *name, const char *no_proxy)
+{
+  /* no_proxy=domain1.dom,host.domain2.dom
+   *   (a comma-separated list of hosts which should
+   *   not be proxied, or an asterisk to override
+   *   all proxy variables)
+   */
+  if(no_proxy && no_proxy[0]) {
+    const char *p = no_proxy;
+    size_t namelen;
+    enum nametype type = TYPE_HOST;
+    char hostip[128];
+    if(!strcmp("*", no_proxy))
+      return TRUE;
+
+    /* NO_PROXY was specified and it wasn't just an asterisk */
+
+    if(name[0] == '[') {
+      char *endptr;
+      /* IPv6 numerical address */
+      endptr = strchr(name, ']');
+      if(!endptr)
+        return FALSE;
+      name++;
+      namelen = endptr - name;
+      if(namelen >= sizeof(hostip))
+        return FALSE;
+      memcpy(hostip, name, namelen);
+      hostip[namelen] = 0;
+      name = hostip;
+      type = TYPE_IPV6;
+    }
+    else {
+      unsigned int address;
+      if(1 == Curl_inet_pton(AF_INET, name, &address))
+        type = TYPE_IPV4;
+      namelen = strlen(name);
+    }
+
+    while(*p) {
+      const char *token;
+      size_t tokenlen = 0;
+      bool match = FALSE;
+
+      /* pass blanks */
+      while(*p && ISBLANK(*p))
+        p++;
+
+      token = p;
+      /* pass over the pattern */
+      while(*p && !ISBLANK(*p) && (*p != ',')) {
+        p++;
+        tokenlen++;
+      }
+
+      if(tokenlen) {
+        switch(type) {
+        case TYPE_HOST:
+          if(*token == '.') {
+            ++token;
+            --tokenlen;
+            /* tailmatch */
+            match = (tokenlen <= namelen) &&
+              strncasecompare(token, name + (namelen - tokenlen), namelen);
+          }
+          else
+            match = (tokenlen == namelen) &&
+              strncasecompare(token, name, namelen);
+          break;
+        case TYPE_IPV4:
+          /* FALLTHROUGH */
+        case TYPE_IPV6: {
+          const char *check = token;
+          char *slash = strchr(check, '/');
+          unsigned int bits = 0;
+          char checkip[128];
+          /* if the slash is part of this token, use it */
+          if(slash && (slash < &check[tokenlen])) {
+            bits = atoi(slash + 1);
+            /* copy the check name to a temp buffer */
+            if(tokenlen >= sizeof(checkip))
+              break;
+            memcpy(checkip, check, tokenlen);
+            checkip[ slash - check ] = 0;
+            check = checkip;
+          }
+          if(type == TYPE_IPV6)
+            match = Curl_cidr6_match(name, check, bits);
+          else
+            match = Curl_cidr4_match(name, check, bits);
+          break;
+        }
+        }
+        if(match)
+          return TRUE;
+      } /* if(tokenlen) */
+      while(*p == ',')
+        p++;
+    } /* while(*p) */
+  } /* NO_PROXY was specified and it wasn't just an asterisk */
+
+  return FALSE;
+}
+
+#endif /* CURL_DISABLE_PROXY */
+

lib/noproxy.h

@@ -0,0 +1,44 @@
+#ifndef HEADER_CURL_NOPROXY_H
+#define HEADER_CURL_NOPROXY_H
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * SPDX-License-Identifier: curl
+ *
+ ***************************************************************************/
+#include "curl_setup.h"
+
+#ifndef CURL_DISABLE_PROXY
+
+#ifdef DEBUGBUILD
+
+UNITTEST bool Curl_cidr4_match(const char *ipv4,    /* 1.2.3.4 address */
+                               const char *network, /* 1.2.3.4 address */
+                               unsigned int bits);
+UNITTEST bool Curl_cidr6_match(const char *ipv6,
+                               const char *network,
+                               unsigned int bits);
+#endif
+
+bool Curl_check_noproxy(const char *name, const char *no_proxy);
+
+#endif
+
+#endif /* HEADER_CURL_NOPROXY_H */

lib/url.c

@@ -106,6 +106,7 @@ bool Curl_win32_idn_to_ascii(const char *in, char **out);
 #include "urlapi-int.h"
 #include "system_win32.h"
 #include "hsts.h"
+#include "noproxy.h"
 
 /* And now for the protocols */
 #include "ftp.h"
@@ -2261,83 +2262,6 @@ void Curl_free_request_state(struct Curl_easy *data)
 
 
 #ifndef CURL_DISABLE_PROXY
-/****************************************************************
-* Checks if the host is in the noproxy list. returns true if it matches
-* and therefore the proxy should NOT be used.
-****************************************************************/
-static bool check_noproxy(const char *name, const char *no_proxy)
-{
-  /* no_proxy=domain1.dom,host.domain2.dom
-   *   (a comma-separated list of hosts which should
-   *   not be proxied, or an asterisk to override
-   *   all proxy variables)
-   */
-  if(no_proxy && no_proxy[0]) {
-    size_t tok_start;
-    size_t tok_end;
-    const char *separator = ", ";
-    size_t no_proxy_len;
-    size_t namelen;
-    char *endptr;
-    if(strcasecompare("*", no_proxy)) {
-      return TRUE;
-    }
-
-    /* NO_PROXY was specified and it wasn't just an asterisk */
-
-    no_proxy_len = strlen(no_proxy);
-    if(name[0] == '[') {
-      /* IPv6 numerical address */
-      endptr = strchr(name, ']');
-      if(!endptr)
-        return FALSE;
-      name++;
-      namelen = endptr - name;
-    }
-    else
-      namelen = strlen(name);
-
-    for(tok_start = 0; tok_start < no_proxy_len; tok_start = tok_end + 1) {
-      while(tok_start < no_proxy_len &&
-            strchr(separator, no_proxy[tok_start]) != NULL) {
-        /* Look for the beginning of the token. */
-        ++tok_start;
-      }
-
-      if(tok_start == no_proxy_len)
-        break; /* It was all trailing separator chars, no more tokens. */
-
-      for(tok_end = tok_start; tok_end < no_proxy_len &&
-            strchr(separator, no_proxy[tok_end]) == NULL; ++tok_end)
-        /* Look for the end of the token. */
-        ;
-
-      /* To match previous behavior, where it was necessary to specify
-       * ".local.com" to prevent matching "notlocal.com", we will leave
-       * the '.' off.
-       */
-      if(no_proxy[tok_start] == '.')
-        ++tok_start;
-
-      if((tok_end - tok_start) <= namelen) {
-        /* Match the last part of the name to the domain we are checking. */
-        const char *checkn = name + namelen - (tok_end - tok_start);
-        if(strncasecompare(no_proxy + tok_start, checkn,
-                           tok_end - tok_start)) {
-          if((tok_end - tok_start) == namelen || *(checkn - 1) == '.') {
-            /* We either have an exact match, or the previous character is a .
-             * so it is within the same domain, so no proxy for this host.
-             */
-            return TRUE;
-          }
-        }
-      } /* if((tok_end - tok_start) <= namelen) */
-    } /* for(tok_start = 0; tok_start < no_proxy_len;
-         tok_start = tok_end + 1) */
-  } /* NO_PROXY was specified and it wasn't just an asterisk */
-
-  return FALSE;
-}
 
 #ifndef CURL_DISABLE_HTTP
 /****************************************************************
@@ -2706,8 +2630,8 @@ static CURLcode create_conn_helper_init_proxy(struct Curl_easy *data,
     }
   }
 
-  if(check_noproxy(conn->host.name, data->set.str[STRING_NOPROXY] ?
-      data->set.str[STRING_NOPROXY] : no_proxy)) {
+  if(Curl_check_noproxy(conn->host.name, data->set.str[STRING_NOPROXY] ?
+                        data->set.str[STRING_NOPROXY] : no_proxy)) {
     Curl_safefree(proxy);
     Curl_safefree(socksproxy);
   }

tests/data/Makefile.inc

@@ -202,7 +202,7 @@ test1566 test1567 test1568 test1569 test1570 \
 test1590 test1591 test1592 test1593 test1594 test1595 test1596 test1597 \
 \
 test1600 test1601 test1602 test1603 test1604 test1605 test1606 test1607 \
-test1608 test1609 test1610 test1611 test1612 test1613 \
+test1608 test1609 test1610 test1611 test1612 test1613 test1614 \
 \
 test1620 test1621 \
 \

tests/data/test1614

@@ -0,0 +1,25 @@
+<testcase>
+<info>
+<keywords>
+unittest
+</keywords>
+</info>
+
+#
+# Client-side
+<client>
+<server>
+none
+</server>
+<features>
+unittest
+proxy
+</features>
+ <name>
+cidr comparisons
+ </name>
+</client>
+<errorcode>
+0
+</errorcode>
+</testcase>

tests/unit/Makefile.inc

@@ -34,7 +34,7 @@ UNITPROGS = unit1300 unit1301 unit1302 unit1303 unit1304 unit1305 unit1307 \
  unit1330 unit1394 unit1395 unit1396 unit1397 unit1398 \
  unit1399 \
  unit1600 unit1601 unit1602 unit1603 unit1604 unit1605 unit1606 unit1607 \
- unit1608 unit1609 unit1610 unit1611 unit1612 \
+ unit1608 unit1609 unit1610 unit1611 unit1612 unit1614 \
  unit1620 unit1621 \
  unit1650 unit1651 unit1652 unit1653 unit1654 unit1655 \
  unit1660 unit1661
@@ -132,6 +132,9 @@ unit1611_CPPFLAGS = $(AM_CPPFLAGS)
 unit1612_SOURCES = unit1612.c $(UNITFILES)
 unit1612_CPPFLAGS = $(AM_CPPFLAGS)
 
+unit1614_SOURCES = unit1614.c $(UNITFILES)
+unit1614_CPPFLAGS = $(AM_CPPFLAGS)
+
 unit1620_SOURCES = unit1620.c $(UNITFILES)
 unit1620_CPPFLAGS = $(AM_CPPFLAGS)
 

tests/unit/unit1614.c

@@ -0,0 +1,133 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * SPDX-License-Identifier: curl
+ *
+ ***************************************************************************/
+#include "curlcheck.h"
+
+#include "noproxy.h"
+
+static CURLcode unit_setup(void)
+{
+  return CURLE_OK;
+}
+
+static void unit_stop(void)
+{
+
+}
+
+struct check {
+  const char *a;
+  const char *n;
+  unsigned int bits;
+  bool match;
+};
+
+struct noproxy {
+  const char *a;
+  const char *n;
+  bool match;
+};
+
+UNITTEST_START
+#ifdef DEBUGBUILD
+{
+  int i;
+  int err = 0;
+  struct check list4[]= {
+    { "192.160.0.1", "192.160.0.1", 33, FALSE},
+    { "192.160.0.1", "192.160.0.1", 32, TRUE},
+    { "192.160.0.1", "192.160.0.1", 0, TRUE},
+    { "192.160.0.1", "192.160.0.1", 24, TRUE},
+    { "192.160.0.1", "192.160.0.1", 26, TRUE},
+    { "192.160.0.1", "192.160.0.1", 20, TRUE},
+    { "192.160.0.1", "192.160.0.1", 18, TRUE},
+    { "192.160.0.1", "192.160.0.1", 12, TRUE},
+    { "192.160.0.1", "192.160.0.1", 8, TRUE},
+    { "192.160.0.1", "10.0.0.1", 8, FALSE},
+    { "192.160.0.1", "10.0.0.1", 32, FALSE},
+    { "192.160.0.1", "10.0.0.1", 0, FALSE},
+    { NULL, NULL, 0, FALSE} /* end marker */
+  };
+  struct check list6[]= {
+    { "::1", "::1", 0, TRUE},
+    { "::1", "::1", 128, TRUE},
+    { "::1", "0:0::1", 128, TRUE},
+    { "::1", "0:0::1", 129, FALSE},
+    { "fe80::ab47:4396:55c9:8474", "fe80::ab47:4396:55c9:8474", 64, TRUE},
+    { NULL, NULL, 0, FALSE} /* end marker */
+  };
+  struct noproxy list[]= {
+    { "foobar", "barfoo", FALSE},
+    { "foobar", "foobar", TRUE},
+    { "192.168.0.1", "foobar", FALSE},
+    { "192.168.0.1", "192.168.0.0/16", TRUE},
+    { "192.168.0.1", "192.168.0.0/24", TRUE},
+    { "192.168.0.1", "192.168.0.0/32", FALSE},
+    { "192.168.0.1", "192.168.0.0", FALSE},
+    { "192.168.1.1", "192.168.0.0/24", FALSE},
+    { "192.168.1.1", "foo, bar, 192.168.0.0/24", FALSE},
+    { "192.168.1.1", "foo, bar, 192.168.0.0/16", TRUE},
+    { "[::1]", "foo, bar, 192.168.0.0/16", FALSE},
+    { "[::1]", "foo, bar, ::1/64", TRUE},
+    { "bar", "foo, bar, ::1/64", TRUE},
+    { "BAr", "foo, bar, ::1/64", TRUE},
+    { "BAr", "foo,,,,,              bar, ::1/64", TRUE},
+    { "www.example.com", "foo, .example.com", TRUE},
+    { "www.example.com", "www2.example.com, .example.net", FALSE},
+    { "example.com", ".example.com, .example.net", TRUE},
+    { "nonexample.com", ".example.com, .example.net", FALSE},
+    { NULL, NULL, FALSE}
+  };
+  for(i = 0; list4[i].a; i++) {
+    bool match = Curl_cidr4_match(list4[i].a, list4[i].n, list4[i].bits);
+    if(match != list4[i].match) {
+      fprintf(stderr, "%s in %s/%u should %smatch\n",
+              list4[i].a, list4[i].n, list4[i].bits,
+              list4[i].match ? "": "not ");
+      err++;
+    }
+  }
+  for(i = 0; list6[i].a; i++) {
+    bool match = Curl_cidr6_match(list6[i].a, list6[i].n, list6[i].bits);
+    if(match != list6[i].match) {
+      fprintf(stderr, "%s in %s/%u should %smatch\n",
+              list6[i].a, list6[i].n, list6[i].bits,
+              list6[i].match ? "": "not ");
+      err++;
+    }
+  }
+  for(i = 0; list[i].a; i++) {
+    bool match = Curl_check_noproxy(list[i].a, list[i].n);
+    if(match != list[i].match) {
+      fprintf(stderr, "%s in %s should %smatch\n",
+              list[i].a, list[i].n,
+              list[i].match ? "": "not ");
+      err++;
+    }
+  }
+  return err;
+}
+#else
+return 0;
+#endif
+UNITTEST_STOP