mem-include-scan: verify memory #includes

If we use memory functions (malloc, free, strdup etc) in C sources in
libcurl and we fail to include curl_memory.h or memdebug.h we either
fail to properly support user-provided memory callbacks or the memory
leak system of the test suite fails.

After Ajit's report of a failure in the first category in http_proxy.c,
I spotted a few in the second category as well. These problems are now
tested for by test 1132 which runs a perl program that scans for and
attempts to check that we use the correct include files if a memory
related function is used in the source code.

Reported by: Ajit Dhumale
Bug: http://curl.haxx.se/mail/lib-2012-11/0125.html

lib/dict.c

@@ -67,10 +67,10 @@
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
 
+#include "curl_memory.h"
 /* The last #include file should be: */
 #include "memdebug.h"
 
-
 /*
  * Forward declarations.
  */

lib/gopher.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -70,10 +70,10 @@
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
 
+#include "curl_memory.h"
 /* The last #include file should be: */
 #include "memdebug.h"
 
-
 /*
  * Forward declarations.
  */

lib/http_proxy.c

@@ -45,6 +45,7 @@
 
 #include "curlx.h"
 
+#include "curl_memory.h"
 /* The last #include file should be: */
 #include "memdebug.h"
 

lib/idn_win32.c

@@ -30,6 +30,10 @@
 
 #include "curl_multibyte.h"
 
+#include "curl_memory.h"
+/* The last #include file should be: */
+#include "memdebug.h"
+
 #ifdef WANT_IDN_PROTOTYPES
 WINBASEAPI int WINAPI IdnToAscii(DWORD, const WCHAR *, int, WCHAR *, int);
 WINBASEAPI int WINAPI IdnToUnicode(DWORD, const WCHAR *, int, WCHAR *, int);

lib/md5.c

@@ -28,9 +28,13 @@
 #include "curl_hmac.h"
 #include "warnless.h"
 
+#include "curl_memory.h"
+
 #if defined(USE_GNUTLS_NETTLE)
 
 #include <nettle/md5.h>
+/* The last #include file should be: */
+#include "memdebug.h"
 
 typedef struct md5_ctx MD5_CTX;
 
@@ -54,6 +58,8 @@ static void MD5_Final(unsigned char digest[16], MD5_CTX * ctx)
 #elif defined(USE_GNUTLS)
 
 #include <gcrypt.h>
+/* The last #include file should be: */
+#include "memdebug.h"
 
 typedef gcry_md_hd_t MD5_CTX;
 
@@ -436,6 +442,9 @@ static void Decode (UINT4 *output,
 
 #endif /* CRYPTO LIBS */
 
+/* The last #include file should be: */
+#include "memdebug.h"
+
 const HMAC_params Curl_HMAC_MD5[] = {
   {
     (HMAC_hinit_func) MD5_Init,           /* Hash initialization function. */

lib/non-ascii.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -24,12 +24,16 @@
 
 #ifdef CURL_DOES_CONVERSIONS
 
+#include <curl/curl.h>
+
 #include "non-ascii.h"
 #include "formdata.h"
 #include "sendf.h"
 #include "urldata.h"
 
-#include <curl/curl.h>
+#include "curl_memory.h"
+/* The last #include file should be: */
+#include "memdebug.h"
 
 #ifdef HAVE_ICONV
 #include <iconv.h>

lib/nwlib.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2009, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -32,6 +32,9 @@
 #include <nks/thread.h>
 #include <nks/synch.h>
 
+#include "curl_memory.h"
+/* The last #include file should be: */
+#include "memdebug.h"
 
 typedef struct
 {

lib/strdup.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2008, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -19,7 +19,9 @@
  * KIND, either express or implied.
  *
  ***************************************************************************/
-
+/*
+ * This file is 'mem-include-scan' clean. See test 1132.
+ */
 #include "setup.h"
 
 #include "strdup.h"

lib/strerror.c

@@ -44,6 +44,9 @@
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
 
+#include "curl_memory.h"
+/* The last #include file should be: */
+#include "memdebug.h"
 
 const char *
 curl_easy_strerror(CURLcode error)

tests/Makefile.am

@@ -28,7 +28,7 @@ EXTRA_DIST = ftpserver.pl httpserver.pl secureserver.pl runtests.pl getpart.pm \
  sshserver.pl sshhelp.pm testcurl.1 runtests.1 $(HTMLPAGES) $(PDFPAGES) \
  CMakeLists.txt certs/scripts/*.sh certs/Server* certs/EdelCurlRoot* \
  serverhelp.pm tftpserver.pl rtspserver.pl directories.pm symbol-scan.pl \
- certs/srp-verifier-conf certs/srp-verifier-db
+ certs/srp-verifier-conf certs/srp-verifier-db mem-include-scan.pl
 
 # we have two variables here to make sure DIST_SUBDIRS won't get 'unit'
 # added twice as then targets such as 'distclean' misbehave and try to

tests/data/Makefile.am

@@ -75,7 +75,7 @@ test1094 test1095 test1096 test1097 test1098 test1099 test1100 test1101	\
 test1102 test1103 test1104 test1105 test1106 test1107 test1108 test1109	\
 test1110 test1111 test1112 test1113 test1114 test1115 test1116 test1117	\
 test1118 test1119 test1120 test1121 test1122 test1123 test1124 test1125	\
-test1126 test1127 test1128 test1129 test1130 test1131 \
+test1126 test1127 test1128 test1129 test1130 test1131 test1132 \
 test1200 test1201 test1202 test1203 test1204 test1205 test1206 test1207 \
 test1208 test1209 test1210 test1211 \
 test1220 \

tests/data/test1132

@@ -0,0 +1,24 @@
+<testcase>
+<info>
+<keywords>
+memory-includes
+</keywords>
+</info>
+
+#
+# Client-side
+<client>
+<server>
+none
+</server>
+
+ <name>
+Verify memory #include files in libcurl's C source files
+ </name>
+
+<command type="perl">
+%SRCDIR/mem-include-scan.pl %SRCDIR/../lib
+</command>
+</client>
+
+</testcase>

tests/mem-include-scan.pl

@@ -0,0 +1,96 @@
+#!/usr/bin/env perl
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) 2010-2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at http://curl.haxx.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+###########################################################################
+#
+# This script scans C source files. If they seem to use memory functions,
+# it also makes sure that it #includes the correct two header files!
+#
+# You can also mark a C source as fine by using 'mem-include-scan' anywhere in
+# it.
+#
+
+use strict;
+use warnings;
+
+my $dir = $ARGV[0] || die "specify directory!";
+
+sub scanfile {
+    my ($file) = @_;
+    my $memfunc;
+    my $memdebug;
+    my $curlmem;
+
+    print STDERR "checking $file...\n";
+
+    open(F, "<$file");
+    while(<F>) {
+        if($_ =~ /(free|alloc|strdup)\(/) {
+            $memfunc++;
+        }
+        elsif($_ =~ /^ *# *include \"memdebug.h\"/) {
+            $memdebug++;
+        }
+        elsif($_ =~ /^ *# *include \"curl_memory.h\"/) {
+            $curlmem++;
+        }
+        elsif($_ =~ /mem-include-scan/) {
+            # free pass
+            close(F);
+            return 0;
+        }
+        if($memfunc && $memdebug && $curlmem) {
+            last;
+        }
+    }
+    close(F);
+
+
+    if($memfunc) {
+        if($memdebug && $curlmem) {
+            return 0;
+        }
+        else {
+            if(!$memdebug) {
+                print STDERR "$file doesn't include \"memdebug.h\"!\n";
+            }
+            if(!$curlmem) {
+                print STDERR "$file doesn't include \"curl_memory.h\"!\n";
+            }
+            return 1;
+        }
+    }
+    return 0;
+}
+
+opendir(my $dh, $dir) || die "can't opendir $dir: $!";
+my @cfiles = grep { /\.c\z/ && -f "$dir/$_" } readdir($dh);
+closedir $dh;
+
+my $errs;
+for(@cfiles) {
+    $errs += scanfile("$dir/$_");
+}
+
+if($errs) {
+    print STDERR "----\n$errs errors detected!\n";
+    exit 2;
+}