http: add support to read and store the referrer header

- add CURLINFO_REFERER libcurl option
- add --write-out '%{referer}' command-line option
- extend --xattr command-line option to fill user.xdg.referrer.url extended
  attribute with the referrer (if there was any)

Closes #6591

docs/cmdline-opts/write-out.d

@@ -93,6 +93,9 @@ When an HTTP request was made without --location to follow redirects (or when
 --max-redir is met), this variable will show the actual URL a redirect
 \fIwould\fP have gone to. (Added in 7.18.2)
 .TP
+.B referer
+The Referer: header, if there was any. (Added in 7.76.0)
+.TP
 .B remote_ip
 The remote IP address of the most recently done connection - can be either
 IPv4 or IPv6 (Added in 7.29.0)

docs/examples/Makefile.inc

@@ -43,6 +43,7 @@ check_PROGRAMS = \
   getinfo \
   getinmemory \
   getredirect \
+  getreferrer \
   http-post \
   http2-download \
   http2-pushinmemory \

docs/libcurl/curl_easy_getinfo.3

@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -50,6 +50,9 @@ See \fICURLINFO_EFFECTIVE_URL(3)\fP
 .IP CURLINFO_RESPONSE_CODE
 Last received response code.
 See \fICURLINFO_RESPONSE_CODE(3)\fP
+.IP CURLINFO_REFERER
+Referrer header.
+See \fICURLINFO_REFERER(3)\fP
 .IP CURLINFO_HTTP_CONNECTCODE
 Last proxy CONNECT response code.
 See \fICURLINFO_HTTP_CONNECTCODE(3)\fP

docs/libcurl/opts/CURLOPT_REFERER.3

@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -58,3 +58,4 @@ Returns CURLE_OK if HTTP support is enabled, CURLE_UNKNOWN_OPTION if not, or
 CURLE_OUT_OF_MEMORY if there was insufficient heap space.
 .SH "SEE ALSO"
 .BR CURLOPT_USERAGENT "(3), " CURLOPT_HTTPHEADER "(3), "
+.BR CURLINFO_REFERER "(3), "

docs/libcurl/opts/Makefile.inc

@@ -5,7 +5,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -64,6 +64,7 @@ man_MANS =                                      \
   CURLINFO_REDIRECT_TIME.3                      \
   CURLINFO_REDIRECT_TIME_T.3                    \
   CURLINFO_REDIRECT_URL.3                       \
+  CURLINFO_REFERER.3                            \
   CURLINFO_REQUEST_SIZE.3                       \
   CURLINFO_RESPONSE_CODE.3                      \
   CURLINFO_RETRY_AFTER.3                        \

docs/libcurl/symbols-in-versions

@@ -271,6 +271,7 @@ CURLINFO_REDIRECT_COUNT         7.9.7
 CURLINFO_REDIRECT_TIME          7.9.7
 CURLINFO_REDIRECT_TIME_T        7.61.0
 CURLINFO_REDIRECT_URL           7.18.2
+CURLINFO_REFERER                7.76.0
 CURLINFO_REQUEST_SIZE           7.4.1
 CURLINFO_RESPONSE_CODE          7.10.8
 CURLINFO_RETRY_AFTER            7.66.0

include/curl/curl.h

@@ -2761,8 +2761,9 @@ typedef enum {
   CURLINFO_RETRY_AFTER      = CURLINFO_OFF_T + 57,
   CURLINFO_EFFECTIVE_METHOD = CURLINFO_STRING + 58,
   CURLINFO_PROXY_ERROR      = CURLINFO_LONG + 59,
+  CURLINFO_REFERER          = CURLINFO_STRING + 60,
 
-  CURLINFO_LASTONE          = 59
+  CURLINFO_LASTONE          = 60
 } CURLINFO;
 
 /* CURLINFO_RESPONSE_CODE is the new name for the option previously known as

lib/getinfo.c

@@ -145,6 +145,10 @@ static CURLcode getinfo_char(struct Curl_easy *data, CURLINFO info,
        option had been enabled! */
     *param_charp = data->info.wouldredirect;
     break;
+  case CURLINFO_REFERER:
+    /* Return the referrer header for this request, or NULL if unset */
+    *param_charp = data->change.referer;
+    break;
   case CURLINFO_PRIMARY_IP:
     /* Return the ip address of the most recent (primary) connection */
     *param_charp = data->info.conn_primary_ip;

packages/OS400/README.OS400

@@ -178,6 +178,7 @@ use:
         CURLINFO_CONTENT_TYPE
         CURLINFO_FTP_ENTRY_PATH
         CURLINFO_REDIRECT_URL
+        CURLINFO_REFERER
         CURLINFO_PRIMARY_IP
         CURLINFO_RTSP_SESSION_ID
         CURLINFO_LOCAL_IP

packages/OS400/curl.inc.in

@@ -1690,6 +1690,8 @@
      d                 c                   X'0010003A'
      d  CURLINFO_PROXY_ERROR...                                                 CURLINFO_LONG + 59
      d                 c                   X'0020003B'
+     d  CURLINFO_REFERER...                                                     CURLINFO_STRING + 60
+     d                 c                   X'0010003C'
       *
      d  CURLINFO_HTTP_CODE...                                                   Old ...RESPONSE_CODE
      d                 c                   X'00200002'

src/tool_writeout.c

@@ -83,6 +83,7 @@ static const struct writeoutvar variables[] = {
   {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT,
    CURLINFO_PROXY_SSL_VERIFYRESULT, writeLong},
   {"redirect_url", VAR_REDIRECT_URL, CURLINFO_REDIRECT_URL, writeString},
+  {"referer", VAR_REFERER, CURLINFO_REFERER, writeString},
   {"remote_ip", VAR_PRIMARY_IP, CURLINFO_PRIMARY_IP, writeString},
   {"remote_port", VAR_PRIMARY_PORT, CURLINFO_PRIMARY_PORT, writeLong},
   {"response_code", VAR_HTTP_CODE, CURLINFO_RESPONSE_CODE, writeLong},

src/tool_writeout.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -54,6 +54,7 @@ typedef enum {
   VAR_REDIRECT_COUNT,
   VAR_REDIRECT_TIME,
   VAR_REDIRECT_URL,
+  VAR_REFERER,
   VAR_REQUEST_SIZE,
   VAR_SCHEME,
   VAR_SIZE_DOWNLOAD,

src/tool_xattr.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -44,9 +44,10 @@ static const struct xattr_mapping {
   /* mappings proposed by
    * https://freedesktop.org/wiki/CommonExtendedAttributes/
    */
-  { "user.xdg.origin.url", CURLINFO_EFFECTIVE_URL },
-  { "user.mime_type",      CURLINFO_CONTENT_TYPE },
-  { NULL,                  CURLINFO_NONE } /* last element, abort loop here */
+  { "user.xdg.referrer.url", CURLINFO_REFERER },
+  { "user.xdg.origin.url",   CURLINFO_EFFECTIVE_URL },
+  { "user.mime_type",        CURLINFO_CONTENT_TYPE },
+  { NULL,                    CURLINFO_NONE } /* last element, abort here */
 };
 
 /* returns TRUE if a new URL is returned, that then needs to be freed */

tests/data/test1067

@@ -53,7 +53,7 @@ http
 HTTP Location: following with auto-referer
  </name>
  <command>
-http://%HOSTIP:%HTTPPORT/want/1067 -L --referer "firstone.html;auto"
+http://%HOSTIP:%HTTPPORT/want/1067 --silent --location --referer "firstone.html;auto" --write-out "%{stderr}|%{referer}|"
 </command>
 </client>
 
@@ -73,5 +73,8 @@ Accept: */*
 Referer: http://%HOSTIP:%HTTPPORT/want/1067
 
 </protocol>
+<stderr nonewline="yes">
+|http://%HOSTIP:%HTTPPORT/want/1067|
+</stderr>
 </verify>
 </testcase>

tests/data/test970

@@ -59,7 +59,7 @@ Accept: */*
 
 </protocol>
 <stdout nonewline="yes">
-{"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"log/out970","ftp_entry_path":null,"http_code":200,"http_connect":000,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/970","url_effective":"http://%HOSTIP:%HTTPPORT/970","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
+{"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"log/out970","ftp_entry_path":null,"http_code":200,"http_connect":000,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/970","url_effective":"http://%HOSTIP:%HTTPPORT/970","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>