curl-rustls.m4: add pkg-config support to rustls detection

Based on the existing openssl pkg-config detection, this commit tries to
use pkg-config to find `rustls` then falls back to the current approach
if that fails.

We use the following logic:

- if no path is provided, just use pkg-config, if it's not there we have
  a problem!
- if a path is provided, try pkg-config
  + if pkg-config fails, try and find rustls directly

Closes #13179

m4/curl-rustls.m4

@@ -28,83 +28,150 @@ dnl check for rustls
 dnl ----------------------------------------------------
 
 if test "x$OPT_RUSTLS" != xno; then
-  _cppflags=$CPPFLAGS
-  _ldflags=$LDFLAGS
   ssl_msg=
 
-  if test X"$OPT_RUSTLS" != Xno; then
+  dnl backup the pre-ssl variables
+  CLEANLDFLAGS="$LDFLAGS"
+  CLEANCPPFLAGS="$CPPFLAGS"
 
-    if test "$OPT_RUSTLS" = "yes"; then
-      OPT_RUSTLS=""
-    fi
+  case $host_os in
+    darwin*)
+      LDFLAGS="$LDFLAGS -framework Security"
+      ;;
+    *)
+      ;;
+  esac
+  ## NEW CODE
 
-    case $host_os in
-      darwin*)
-        LDFLAGS="$LDFLAGS -framework Security"
-        ;;
-      *)
-        ;;
-    esac
-
-    if test -z "$OPT_RUSTLS" ; then
-      dnl check for lib first without setting any new path
-
-      AC_CHECK_LIB(rustls, rustls_client_session_read,
-      dnl librustls found, set the variable
-       [
-         AC_DEFINE(USE_RUSTLS, 1, [if rustls is enabled])
-         AC_SUBST(USE_RUSTLS, [1])
-         RUSTLS_ENABLED=1
-         USE_RUSTLS="yes"
-         ssl_msg="rustls"
-         test rustls != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes
-        ], [], -lpthread -ldl -lm)
-    fi
+  dnl use pkg-config unless we have been given a path
+  dnl even then, try pkg-config first
 
-    if test "x$USE_RUSTLS" != "xyes"; then
-      dnl add the path and test again
-      addld=-L$OPT_RUSTLS/lib$libsuff
-      addcflags=-I$OPT_RUSTLS/include
-      rustlslib=$OPT_RUSTLS/lib$libsuff
+  case "$OPT_RUSTLS" in
+    yes)
+      dnl --with-rustls (without path) used
+      PKGTEST="yes"
+      PREFIX_RUSTLS=
+      ;;
+    *)
+      dnl check the provided --with-rustls path
+      PKGTEST="no"
+      PREFIX_RUSTLS=$OPT_RUSTLS
 
-      LDFLAGS="$LDFLAGS $addld"
-      if test "$addcflags" != "-I/usr/include"; then
-         CPPFLAGS="$CPPFLAGS $addcflags"
+      dnl Try pkg-config even when cross-compiling.  Since we
+      dnl specify PKG_CONFIG_LIBDIR we are only looking where
+      dnl the user told us to look
+
+      RUSTLS_PCDIR="$PREFIX_RUSTLS/lib/pkgconfig"
+      if test -f "$RUSTLS_PCDIR/rustls.pc"; then
+        AC_MSG_NOTICE([PKG_CONFIG_LIBDIR will be set to "$RUSTLS_PCDIR"])
+        PKGTEST="yes"
       fi
 
-      AC_CHECK_LIB(rustls, rustls_connection_read,
-       [
-       AC_DEFINE(USE_RUSTLS, 1, [if rustls is enabled])
-       AC_SUBST(USE_RUSTLS, [1])
-       RUSTLS_ENABLED=1
-       USE_RUSTLS="yes"
-       ssl_msg="rustls"
-       test rustls != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes
-       ],
-       AC_MSG_ERROR([--with-rustls was specified but could not find rustls.]),
-       -lpthread -ldl -lm)
-    fi
+      if test "$PKGTEST" != "yes"; then
+        # try lib64 instead
+        RUSTLS_PCDIR="$PREFIX_RUSTLS/lib64/pkgconfig"
+        if test -f "$RUSTLS_PCDIR/rustls.pc"; then
+          AC_MSG_NOTICE([PKG_CONFIG_LIBDIR will be set to "$RUSTLS_PCDIR"])
+          PKGTEST="yes"
+        fi
+      fi
+
+      if test "$PKGTEST" != "yes"; then
+        dnl pkg-config came up empty, use what we got
+        dnl via --with-rustls
 
-    if test "x$USE_RUSTLS" = "xyes"; then
-      AC_MSG_NOTICE([detected rustls])
-      check_for_ca_bundle=1
-
-      LIBS="-lrustls -lpthread -ldl -lm $LIBS"
-
-      if test -n "$rustlslib"; then
-        dnl when shared libs were found in a path that the run-time
-        dnl linker doesn't search through, we need to add it to
-        dnl CURL_LIBRARY_PATH to prevent further configure tests to fail
-        dnl due to this
-        if test "x$cross_compiling" != "xyes"; then
-          CURL_LIBRARY_PATH="$CURL_LIBRARY_PATH:$rustlslib"
-          export CURL_LIBRARY_PATH
-          AC_MSG_NOTICE([Added $rustlslib to CURL_LIBRARY_PATH])
+        addld=-L$PREFIX_RUSTLS/lib$libsuff
+        addcflags=-I$PREFIX_RUSTLS/include
+
+        LDFLAGS="$LDFLAGS $addld"
+        if test "$addcflags" != "-I/usr/include"; then
+            CPPFLAGS="$CPPFLAGS $addcflags"
+        fi
+
+        AC_CHECK_LIB(rustls, rustls_connection_read,
+          [
+          AC_DEFINE(USE_RUSTLS, 1, [if rustls is enabled])
+          AC_SUBST(USE_RUSTLS, [1])
+          RUSTLS_ENABLED=1
+          USE_RUSTLS="yes"
+          ssl_msg="rustls"
+          test rustls != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes
+          ],
+          AC_MSG_ERROR([--with-rustls was specified but could not find rustls.]),
+          -lpthread -ldl -lm)
+
+        USE_RUSTLS="yes"
+        ssl_msg="rustls"
+
+        LIB_RUSTLS="$PREFIX_RUSTLS/lib$libsuff"
+        if test "$PREFIX_RUSTLS" != "/usr" ; then
+          SSL_LDFLAGS="-L$LIB_RUSTLS"
+          SSL_CPPFLAGS="-I$PREFIX_RUSTLS/include"
         fi
       fi
+      ;;
+  esac
+
+  if test "$PKGTEST" = "yes"; then
+
+    CURL_CHECK_PKGCONFIG(rustls, [$RUSTLS_PCDIR])
+
+    if test "$PKGCONFIG" != "no" ; then
+      SSL_LIBS=`CURL_EXPORT_PCDIR([$RUSTLS_PCDIR]) dnl
+        $PKGCONFIG --libs-only-l --libs-only-other rustls 2>/dev/null`
+
+      SSL_LDFLAGS=`CURL_EXPORT_PCDIR([$RUSTLS_PCDIR]) dnl
+        $PKGCONFIG --libs-only-L rustls 2>/dev/null`
+
+      SSL_CPPFLAGS=`CURL_EXPORT_PCDIR([$RUSTLS_PCDIR]) dnl
+        $PKGCONFIG --cflags-only-I rustls 2>/dev/null`
+
+      AC_SUBST(SSL_LIBS)
+      AC_MSG_NOTICE([pkg-config: SSL_LIBS: "$SSL_LIBS"])
+      AC_MSG_NOTICE([pkg-config: SSL_LDFLAGS: "$SSL_LDFLAGS"])
+      AC_MSG_NOTICE([pkg-config: SSL_CPPFLAGS: "$SSL_CPPFLAGS"])
+
+      LIB_RUSTLS=`echo $SSL_LDFLAGS | sed -e 's/^-L//'`
+
+      dnl use the values pkg-config reported.  This is here
+      dnl instead of below with CPPFLAGS and LDFLAGS because we only
+      dnl learn about this via pkg-config.  If we only have
+      dnl the argument to --with-rustls we don't know what
+      dnl additional libs may be necessary.  Hope that we
+      dnl don't need any.
+      LIBS="$SSL_LIBS $LIBS"
+      USE_RUSTLS="yes"
+      ssl_msg="rustls"
+    else
+      AC_MSG_ERROR([pkg-config: Could not find rustls])
     fi
 
-  fi dnl rustls not disabled
+  else
+    dnl we did not use pkg-config, so we need to add the
+    dnl rustls lib to LIBS
+    LIBS="-lrustls -lpthread -ldl -lm $LIBS"
+  fi
+
+  dnl finally, set flags to use this TLS backend
+  CPPFLAGS="$CLEAN_CPPFLAGS $SSL_CPPFLAGS"
+  LDFLAGS="$CLAN_LDFLAGS $SSL_LDFLAGS"
+
+  if test "x$USE_RUSTLS" = "xyes"; then
+    AC_MSG_NOTICE([detected rustls])
+    check_for_ca_bundle=1
+
+    if test -n "$LIB_RUSTLS"; then
+      dnl when shared libs were found in a path that the run-time
+      dnl linker does not search through, we need to add it to
+      dnl CURL_LIBRARY_PATH so that further configure tests do not
+      dnl fail due to this
+      if test "x$cross_compiling" != "xyes"; then
+        CURL_LIBRARY_PATH="$CURL_LIBRARY_PATH:$LIB_RUSTLS"
+        export CURL_LIBRARY_PATH
+        AC_MSG_NOTICE([Added $LIB_RUSTLS to CURL_LIBRARY_PATH])
+      fi
+    fi
+  fi
 
   test -z "$ssl_msg" || ssl_backends="${ssl_backends:+$ssl_backends, }$ssl_msg"
 fi