runtests.pl 99 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038
  1. #!/usr/bin/env perl
  2. #***************************************************************************
  3. # _ _ ____ _
  4. # Project ___| | | | _ \| |
  5. # / __| | | | |_) | |
  6. # | (__| |_| | _ <| |___
  7. # \___|\___/|_| \_\_____|
  8. #
  9. # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  10. #
  11. # This software is licensed as described in the file COPYING, which
  12. # you should have received as part of this distribution. The terms
  13. # are also available at https://curl.se/docs/copyright.html.
  14. #
  15. # You may opt to use, copy, modify, merge, publish, distribute and/or sell
  16. # copies of the Software, and permit persons to whom the Software is
  17. # furnished to do so, under the terms of the COPYING file.
  18. #
  19. # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  20. # KIND, either express or implied.
  21. #
  22. # SPDX-License-Identifier: curl
  23. #
  24. ###########################################################################
  25. # Experimental hooks are available to run tests remotely on machines that
  26. # are able to run curl but are unable to run the test harness.
  27. # The following sections need to be modified:
  28. #
  29. # $HOSTIP, $HOST6IP - Set to the address of the host running the test suite
  30. # $CLIENTIP, $CLIENT6IP - Set to the address of the host running curl
  31. # runclient, runclientoutput - Modify to copy all the files in the log/
  32. # directory to the system running curl, run the given command remotely
  33. # and save the return code or returned stdout (respectively), then
  34. # copy all the files from the remote system's log/ directory back to
  35. # the host running the test suite. This can be done a few ways, such
  36. # as using scp & ssh, rsync & telnet, or using a NFS shared directory
  37. # and ssh.
  38. #
  39. # 'make && make test' needs to be done on both machines before making the
  40. # above changes and running runtests.pl manually. In the shared NFS case,
  41. # the contents of the tests/server/ directory must be from the host
  42. # running the test suite, while the rest must be from the host running curl.
  43. #
  44. # Note that even with these changes a number of tests will still fail (mainly
  45. # to do with cookies, those that set environment variables, or those that
  46. # do more than touch the file system in a <precheck> or <postcheck>
  47. # section). These can be added to the $TESTCASES line below,
  48. # e.g. $TESTCASES="!8 !31 !63 !cookies..."
  49. #
  50. # Finally, to properly support -g and -n, checktestcmd needs to change
  51. # to check the remote system's PATH, and the places in the code where
  52. # the curl binary is read directly to determine its type also need to be
  53. # fixed. As long as the -g option is never given, and the -n is always
  54. # given, this won't be a problem.
  55. use strict;
  56. # Promote all warnings to fatal
  57. use warnings FATAL => 'all';
  58. use 5.006;
  59. # These should be the only variables that might be needed to get edited:
  60. BEGIN {
  61. # Define srcdir to the location of the tests source directory. This is
  62. # usually set by the Makefile, but for out-of-tree builds with direct
  63. # invocation of runtests.pl, it may not be set.
  64. if(!defined $ENV{'srcdir'}) {
  65. use File::Basename;
  66. $ENV{'srcdir'} = dirname(__FILE__);
  67. }
  68. push(@INC, $ENV{'srcdir'});
  69. # run time statistics needs Time::HiRes
  70. eval {
  71. no warnings "all";
  72. require Time::HiRes;
  73. import Time::HiRes qw( time );
  74. }
  75. }
  76. use Digest::MD5 qw(md5);
  77. use List::Util 'sum';
  78. use pathhelp qw(
  79. exe_ext
  80. sys_native_current_path
  81. );
  82. use processhelp qw(
  83. portable_sleep
  84. );
  85. use appveyor;
  86. use azure;
  87. use getpart; # array functions
  88. use servers;
  89. use valgrind; # valgrind report parser
  90. use globalconfig;
  91. use runner;
  92. use testutil;
  93. my %custom_skip_reasons;
  94. my $ACURL=$VCURL; # what curl binary to use to talk to APIs (relevant for CI)
  95. # ACURL is handy to set to the system one for reliability
  96. my $CURLCONFIG="../curl-config"; # curl-config from current build
  97. # Normally, all test cases should be run, but at times it is handy to
  98. # simply run a particular one:
  99. my $TESTCASES="all";
  100. # To run specific test cases, set them like:
  101. # $TESTCASES="1 2 3 7 8";
  102. #######################################################################
  103. # No variables below this point should need to be modified
  104. #
  105. my $libtool;
  106. my $repeat = 0;
  107. my $start; # time at which testing started
  108. my $uname_release = `uname -r`;
  109. my $is_wsl = $uname_release =~ /Microsoft$/;
  110. my $http_ipv6; # set if HTTP server has IPv6 support
  111. my $http_unix; # set if HTTP server has Unix sockets support
  112. my $ftp_ipv6; # set if FTP server has IPv6 support
  113. my $resolver; # name of the resolver backend (for human presentation)
  114. my $has_textaware; # set if running on a system that has a text mode concept
  115. # on files. Windows for example
  116. my %skipped; # skipped{reason}=counter, reasons for skip
  117. my @teststat; # teststat[testnum]=reason, reasons for skip
  118. my %disabled_keywords; # key words of tests to skip
  119. my %ignored_keywords; # key words of tests to ignore results
  120. my %enabled_keywords; # key words of tests to run
  121. my %disabled; # disabled test cases
  122. my %ignored; # ignored results of test cases
  123. my %ignoretestcodes; # if test results are to be ignored
  124. my $timestats; # time stamping and stats generation
  125. my $fullstats; # show time stats for every single test
  126. my %timeprepini; # timestamp for each test preparation start
  127. my %timesrvrini; # timestamp for each test required servers verification start
  128. my %timesrvrend; # timestamp for each test required servers verification end
  129. my %timetoolini; # timestamp for each test command run starting
  130. my %timetoolend; # timestamp for each test command run stopping
  131. my %timesrvrlog; # timestamp for each test server logs lock removal
  132. my %timevrfyend; # timestamp for each test result verification end
  133. my $globalabort; # flag signalling program abort
  134. # values for $singletest_state
  135. use constant {
  136. ST_INIT => 0,
  137. ST_CLEARLOCKS => 1,
  138. ST_INITED => 2,
  139. ST_PREPROCESS => 3,
  140. ST_RUN => 4,
  141. };
  142. my %singletest_state; # current state of singletest() by runner ID
  143. my %singletest_logs; # log messages while in singletest array ref by runner
  144. my $singletest_bufferedrunner; # runner ID which is buffering logs
  145. my %runnerids; # runner IDs by number
  146. my @runnersidle; # runner IDs idle and ready to execute a test
  147. my %countforrunner; # test count by runner ID
  148. my %runnersrunning; # tests currently running by runner ID
  149. #######################################################################
  150. # variables that command line options may set
  151. #
  152. my $short;
  153. my $no_debuginfod;
  154. my $keepoutfiles; # keep stdout and stderr files after tests
  155. my $clearlocks; # force removal of files by killing locking processes
  156. my $postmortem; # display detailed info about failed tests
  157. my $run_disabled; # run the specific tests even if listed in DISABLED
  158. my $scrambleorder;
  159. my $jobs = 0;
  160. # Azure Pipelines specific variables
  161. my $AZURE_RUN_ID = 0;
  162. my $AZURE_RESULT_ID = 0;
  163. #######################################################################
  164. # logmsg is our general message logging subroutine.
  165. #
  166. sub logmsg {
  167. if($singletest_bufferedrunner) {
  168. # Logs are currently being buffered
  169. return singletest_logmsg(@_);
  170. }
  171. for(@_) {
  172. my $line = $_;
  173. if(!$line) {
  174. next;
  175. }
  176. if ($is_wsl) {
  177. # use \r\n for WSL shell
  178. $line =~ s/\r?\n$/\r\n/g;
  179. }
  180. print "$line";
  181. }
  182. }
  183. #######################################################################
  184. # enable logmsg buffering for the given runner ID
  185. #
  186. sub logmsg_bufferfortest {
  187. my ($runnerid)=@_;
  188. if($jobs) {
  189. # Only enable buffering in multiprocess mode
  190. $singletest_bufferedrunner = $runnerid;
  191. }
  192. }
  193. #######################################################################
  194. # Store a log message in a buffer for this test
  195. # The messages can then be displayed all at once at the end of the test
  196. # which prevents messages from different tests from being interleaved.
  197. sub singletest_logmsg {
  198. if(!exists $singletest_logs{$singletest_bufferedrunner}) {
  199. # initialize to a reference to an empty anonymous array
  200. $singletest_logs{$singletest_bufferedrunner} = [];
  201. }
  202. my $logsref = $singletest_logs{$singletest_bufferedrunner};
  203. push @$logsref, @_;
  204. }
  205. #######################################################################
  206. # Stop buffering log messages, but don't touch them
  207. sub singletest_unbufferlogs {
  208. undef $singletest_bufferedrunner;
  209. }
  210. #######################################################################
  211. # Clear the buffered log messages & stop buffering after returning them
  212. sub singletest_dumplogs {
  213. if(!defined $singletest_bufferedrunner) {
  214. # probably not multiprocess mode and logs weren't buffered
  215. return undef;
  216. }
  217. my $logsref = $singletest_logs{$singletest_bufferedrunner};
  218. my $msg = join("", @$logsref);
  219. delete $singletest_logs{$singletest_bufferedrunner};
  220. singletest_unbufferlogs();
  221. return $msg;
  222. }
  223. sub catch_zap {
  224. my $signame = shift;
  225. print "runtests.pl received SIG$signame, exiting\r\n";
  226. $globalabort = 1;
  227. }
  228. $SIG{INT} = \&catch_zap;
  229. $SIG{TERM} = \&catch_zap;
  230. sub catch_usr1 {
  231. print "runtests.pl internal state:\r\n";
  232. print scalar(%runnersrunning) . " busy test runner(s) of " . scalar(keys %runnerids) . "\r\n";
  233. foreach my $rid (sort(keys(%runnersrunning))) {
  234. my $runnernum = "unknown";
  235. foreach my $rnum (keys %runnerids) {
  236. if($runnerids{$rnum} == $rid) {
  237. $runnernum = $rnum;
  238. last;
  239. }
  240. }
  241. print "Runner $runnernum (id $rid) running test $runnersrunning{$rid} in state $singletest_state{$rid}\r\n";
  242. }
  243. }
  244. eval {
  245. # some msys2 perl versions don't define SIGUSR1
  246. $SIG{USR1} = \&catch_usr1;
  247. };
  248. $SIG{PIPE} = 'IGNORE'; # these errors are captured in the read/write calls
  249. ##########################################################################
  250. # Clear all possible '*_proxy' environment variables for various protocols
  251. # to prevent them to interfere with our testing!
  252. foreach my $protocol (('ftp', 'http', 'ftps', 'https', 'no', 'all')) {
  253. my $proxy = "${protocol}_proxy";
  254. # clear lowercase version
  255. delete $ENV{$proxy} if($ENV{$proxy});
  256. # clear uppercase version
  257. delete $ENV{uc($proxy)} if($ENV{uc($proxy)});
  258. }
  259. # make sure we don't get affected by other variables that control our
  260. # behavior
  261. delete $ENV{'SSL_CERT_DIR'} if($ENV{'SSL_CERT_DIR'});
  262. delete $ENV{'SSL_CERT_PATH'} if($ENV{'SSL_CERT_PATH'});
  263. delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'});
  264. # provide defaults from our config file for ENV vars not explicitly
  265. # set by the caller
  266. if (open(my $fd, "<", "config")) {
  267. while(my $line = <$fd>) {
  268. next if ($line =~ /^#/);
  269. chomp $line;
  270. my ($name, $val) = split(/\s*:\s*/, $line, 2);
  271. $ENV{$name} = $val if(!$ENV{$name});
  272. }
  273. close($fd);
  274. }
  275. # Check if we have nghttpx available and if it talks http/3
  276. my $nghttpx_h3 = 0;
  277. if (!$ENV{"NGHTTPX"}) {
  278. $ENV{"NGHTTPX"} = checktestcmd("nghttpx");
  279. }
  280. if ($ENV{"NGHTTPX"}) {
  281. my $nghttpx_version=join(' ', `"$ENV{'NGHTTPX'}" -v 2>/dev/null`);
  282. $nghttpx_h3 = $nghttpx_version =~ /nghttp3\//;
  283. chomp $nghttpx_h3;
  284. }
  285. #######################################################################
  286. # Get the list of tests that the tests/data/Makefile.am knows about!
  287. #
  288. my $disttests = "";
  289. sub get_disttests {
  290. # If a non-default $TESTDIR is being used there may not be any
  291. # Makefile.inc in which case there's nothing to do.
  292. open(my $dh, "<", "$TESTDIR/Makefile.inc") or return;
  293. while(<$dh>) {
  294. chomp $_;
  295. if(($_ =~ /^#/) ||($_ !~ /test/)) {
  296. next;
  297. }
  298. $disttests .= $_;
  299. }
  300. close($dh);
  301. }
  302. #######################################################################
  303. # Remove all files in the specified directory
  304. #
  305. sub cleardir {
  306. my $dir = $_[0];
  307. my $done = 1; # success
  308. my $file;
  309. # Get all files
  310. opendir(my $dh, $dir) ||
  311. return 0; # can't open dir
  312. while($file = readdir($dh)) {
  313. # Don't clear the $PIDDIR or $LOCKDIR since those need to live beyond
  314. # one test
  315. if(($file !~ /^(\.|\.\.)\z/) &&
  316. "$file" ne $PIDDIR && "$file" ne $LOCKDIR) {
  317. if(-d "$dir/$file") {
  318. if(!cleardir("$dir/$file")) {
  319. $done = 0;
  320. }
  321. if(!rmdir("$dir/$file")) {
  322. $done = 0;
  323. }
  324. }
  325. else {
  326. # Ignore stunnel since we cannot do anything about its locks
  327. if(!unlink("$dir/$file") && "$file" !~ /_stunnel\.log$/) {
  328. $done = 0;
  329. }
  330. }
  331. }
  332. }
  333. closedir $dh;
  334. return $done;
  335. }
  336. #######################################################################
  337. # Given two array references, this function will store them in two temporary
  338. # files, run 'diff' on them, store the result and return the diff output!
  339. sub showdiff {
  340. my ($logdir, $firstref, $secondref)=@_;
  341. my $file1="$logdir/check-generated";
  342. my $file2="$logdir/check-expected";
  343. open(my $temp, ">", "$file1") || die "Failure writing diff file";
  344. for(@$firstref) {
  345. my $l = $_;
  346. $l =~ s/\r/[CR]/g;
  347. $l =~ s/\n/[LF]/g;
  348. $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
  349. print $temp $l;
  350. print $temp "\n";
  351. }
  352. close($temp) || die "Failure writing diff file";
  353. open($temp, ">", "$file2") || die "Failure writing diff file";
  354. for(@$secondref) {
  355. my $l = $_;
  356. $l =~ s/\r/[CR]/g;
  357. $l =~ s/\n/[LF]/g;
  358. $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
  359. print $temp $l;
  360. print $temp "\n";
  361. }
  362. close($temp) || die "Failure writing diff file";
  363. my @out = `diff -u $file2 $file1 2>/dev/null`;
  364. if(!$out[0]) {
  365. @out = `diff -c $file2 $file1 2>/dev/null`;
  366. }
  367. return @out;
  368. }
  369. #######################################################################
  370. # compare test results with the expected output, we might filter off
  371. # some pattern that is allowed to differ, output test results
  372. #
  373. sub compare {
  374. my ($runnerid, $testnum, $testname, $subject, $firstref, $secondref)=@_;
  375. my $result = compareparts($firstref, $secondref);
  376. if($result) {
  377. # timestamp test result verification end
  378. $timevrfyend{$testnum} = Time::HiRes::time();
  379. if(!$short) {
  380. logmsg "\n $testnum: $subject FAILED:\n";
  381. my $logdir = getrunnerlogdir($runnerid);
  382. logmsg showdiff($logdir, $firstref, $secondref);
  383. }
  384. elsif(!$automakestyle) {
  385. logmsg "FAILED\n";
  386. }
  387. else {
  388. # automakestyle
  389. logmsg "FAIL: $testnum - $testname - $subject\n";
  390. }
  391. }
  392. return $result;
  393. }
  394. #######################################################################
  395. # Parse and store the protocols in curl's Protocols: line
  396. sub parseprotocols {
  397. my ($line)=@_;
  398. @protocols = split(' ', lc($line));
  399. # Generate a "proto-ipv6" version of each protocol to match the
  400. # IPv6 <server> name and a "proto-unix" to match the variant which
  401. # uses Unix domain sockets. This works even if support isn't
  402. # compiled in because the <features> test will fail.
  403. push @protocols, map(("$_-ipv6", "$_-unix"), @protocols);
  404. # 'http-proxy' is used in test cases to do CONNECT through
  405. push @protocols, 'http-proxy';
  406. # 'none' is used in test cases to mean no server
  407. push @protocols, 'none';
  408. }
  409. #######################################################################
  410. # Check & display information about curl and the host the test suite runs on.
  411. # Information to do with servers is displayed in displayserverfeatures, after
  412. # the server initialization is performed.
  413. sub checksystemfeatures {
  414. my $feat;
  415. my $curl;
  416. my $libcurl;
  417. my $versretval;
  418. my $versnoexec;
  419. my @version=();
  420. my @disabled;
  421. my $dis = "";
  422. my $curlverout="$LOGDIR/curlverout.log";
  423. my $curlvererr="$LOGDIR/curlvererr.log";
  424. my $versioncmd=shell_quote($CURL) . " --version 1>$curlverout 2>$curlvererr";
  425. unlink($curlverout);
  426. unlink($curlvererr);
  427. $versretval = runclient($versioncmd);
  428. $versnoexec = $!;
  429. open(my $versout, "<", "$curlverout");
  430. @version = <$versout>;
  431. close($versout);
  432. open(my $disabledh, "-|", "server/disabled".exe_ext('TOOL'));
  433. @disabled = <$disabledh>;
  434. close($disabledh);
  435. if($disabled[0]) {
  436. s/[\r\n]//g for @disabled;
  437. $dis = join(", ", @disabled);
  438. }
  439. $resolver="stock";
  440. for(@version) {
  441. chomp;
  442. if($_ =~ /^curl ([^ ]*)/) {
  443. $curl = $_;
  444. $CURLVERSION = $1;
  445. $curl =~ s/^(.*)(libcurl.*)/$1/g || die "Failure determining curl binary version";
  446. $libcurl = $2;
  447. if($curl =~ /linux|bsd|solaris/) {
  448. # system support LD_PRELOAD; may be disabled later
  449. $feature{"ld_preload"} = 1;
  450. }
  451. if($curl =~ /win32|Windows|mingw(32|64)/) {
  452. # This is a Windows MinGW build or native build, we need to use
  453. # Win32-style path.
  454. $pwd = sys_native_current_path();
  455. $has_textaware = 1;
  456. $feature{"win32"} = 1;
  457. # set if built with MinGW (as opposed to MinGW-w64)
  458. $feature{"MinGW"} = 1 if ($curl =~ /-pc-mingw32/);
  459. }
  460. if ($libcurl =~ /\s(winssl|schannel)\b/i) {
  461. $feature{"Schannel"} = 1;
  462. $feature{"SSLpinning"} = 1;
  463. }
  464. elsif ($libcurl =~ /\sopenssl\b/i) {
  465. $feature{"OpenSSL"} = 1;
  466. $feature{"SSLpinning"} = 1;
  467. }
  468. elsif ($libcurl =~ /\sgnutls\b/i) {
  469. $feature{"GnuTLS"} = 1;
  470. $feature{"SSLpinning"} = 1;
  471. }
  472. elsif ($libcurl =~ /\srustls-ffi\b/i) {
  473. $feature{"rustls"} = 1;
  474. }
  475. elsif ($libcurl =~ /\swolfssl\b/i) {
  476. $feature{"wolfssl"} = 1;
  477. $feature{"SSLpinning"} = 1;
  478. }
  479. elsif ($libcurl =~ /\sbearssl\b/i) {
  480. $feature{"bearssl"} = 1;
  481. }
  482. elsif ($libcurl =~ /\ssecuretransport\b/i) {
  483. $feature{"sectransp"} = 1;
  484. $feature{"SSLpinning"} = 1;
  485. }
  486. elsif ($libcurl =~ /\sBoringSSL\b/i) {
  487. # OpenSSL compatible API
  488. $feature{"OpenSSL"} = 1;
  489. $feature{"SSLpinning"} = 1;
  490. }
  491. elsif ($libcurl =~ /\slibressl\b/i) {
  492. # OpenSSL compatible API
  493. $feature{"OpenSSL"} = 1;
  494. $feature{"SSLpinning"} = 1;
  495. }
  496. elsif ($libcurl =~ /\smbedTLS\b/i) {
  497. $feature{"mbedtls"} = 1;
  498. $feature{"SSLpinning"} = 1;
  499. }
  500. if ($libcurl =~ /ares/i) {
  501. $feature{"c-ares"} = 1;
  502. $resolver="c-ares";
  503. }
  504. if ($libcurl =~ /Hyper/i) {
  505. $feature{"hyper"} = 1;
  506. }
  507. if ($libcurl =~ /nghttp2/i) {
  508. # nghttp2 supports h2c, hyper does not
  509. $feature{"h2c"} = 1;
  510. }
  511. if ($libcurl =~ /libssh2/i) {
  512. $feature{"libssh2"} = 1;
  513. }
  514. if ($libcurl =~ /libssh\/([0-9.]*)\//i) {
  515. $feature{"libssh"} = 1;
  516. if($1 =~ /(\d+)\.(\d+).(\d+)/) {
  517. my $v = $1 * 100 + $2 * 10 + $3;
  518. if($v < 94) {
  519. # before 0.9.4
  520. $feature{"oldlibssh"} = 1;
  521. }
  522. }
  523. }
  524. if ($libcurl =~ /wolfssh/i) {
  525. $feature{"wolfssh"} = 1;
  526. }
  527. }
  528. elsif($_ =~ /^Protocols: (.*)/i) {
  529. # these are the protocols compiled in to this libcurl
  530. parseprotocols($1);
  531. }
  532. elsif($_ =~ /^Features: (.*)/i) {
  533. $feat = $1;
  534. # built with memory tracking support (--enable-curldebug); may be disabled later
  535. $feature{"TrackMemory"} = $feat =~ /TrackMemory/i;
  536. # curl was built with --enable-debug
  537. $feature{"debug"} = $feat =~ /debug/i;
  538. # ssl enabled
  539. $feature{"SSL"} = $feat =~ /SSL/i;
  540. # multiple ssl backends available.
  541. $feature{"MultiSSL"} = $feat =~ /MultiSSL/i;
  542. # large file support
  543. $feature{"large_file"} = $feat =~ /Largefile/i;
  544. # IDN support
  545. $feature{"idn"} = $feat =~ /IDN/i;
  546. # IPv6 support
  547. $feature{"ipv6"} = $feat =~ /IPv6/i;
  548. # Unix sockets support
  549. $feature{"unix-sockets"} = $feat =~ /UnixSockets/i;
  550. # libz compression
  551. $feature{"libz"} = $feat =~ /libz/i;
  552. # Brotli compression
  553. $feature{"brotli"} = $feat =~ /brotli/i;
  554. # Zstd compression
  555. $feature{"zstd"} = $feat =~ /zstd/i;
  556. # NTLM enabled
  557. $feature{"NTLM"} = $feat =~ /NTLM/i;
  558. # NTLM delegation to winbind daemon ntlm_auth helper enabled
  559. $feature{"NTLM_WB"} = $feat =~ /NTLM_WB/i;
  560. # SSPI enabled
  561. $feature{"SSPI"} = $feat =~ /SSPI/i;
  562. # GSS-API enabled
  563. $feature{"GSS-API"} = $feat =~ /GSS-API/i;
  564. # Kerberos enabled
  565. $feature{"Kerberos"} = $feat =~ /Kerberos/i;
  566. # SPNEGO enabled
  567. $feature{"SPNEGO"} = $feat =~ /SPNEGO/i;
  568. # CharConv enabled
  569. $feature{"CharConv"} = $feat =~ /CharConv/i;
  570. # TLS-SRP enabled
  571. $feature{"TLS-SRP"} = $feat =~ /TLS-SRP/i;
  572. # PSL enabled
  573. $feature{"PSL"} = $feat =~ /PSL/i;
  574. # alt-svc enabled
  575. $feature{"alt-svc"} = $feat =~ /alt-svc/i;
  576. # HSTS support
  577. $feature{"HSTS"} = $feat =~ /HSTS/i;
  578. if($feat =~ /AsynchDNS/i) {
  579. if(!$feature{"c-ares"}) {
  580. # this means threaded resolver
  581. $feature{"threaded-resolver"} = 1;
  582. $resolver="threaded";
  583. }
  584. }
  585. # http2 enabled
  586. $feature{"http/2"} = $feat =~ /HTTP2/;
  587. if($feature{"http/2"}) {
  588. push @protocols, 'http/2';
  589. }
  590. # http3 enabled
  591. $feature{"http/3"} = $feat =~ /HTTP3/;
  592. if($feature{"http/3"}) {
  593. push @protocols, 'http/3';
  594. }
  595. # https proxy support
  596. $feature{"https-proxy"} = $feat =~ /HTTPS-proxy/;
  597. if($feature{"https-proxy"}) {
  598. # 'https-proxy' is used as "server" so consider it a protocol
  599. push @protocols, 'https-proxy';
  600. }
  601. # UNICODE support
  602. $feature{"Unicode"} = $feat =~ /Unicode/i;
  603. # Thread-safe init
  604. $feature{"threadsafe"} = $feat =~ /threadsafe/i;
  605. }
  606. #
  607. # Test harness currently uses a non-stunnel server in order to
  608. # run HTTP TLS-SRP tests required when curl is built with https
  609. # protocol support and TLS-SRP feature enabled. For convenience
  610. # 'httptls' may be included in the test harness protocols array
  611. # to differentiate this from classic stunnel based 'https' test
  612. # harness server.
  613. #
  614. if($feature{"TLS-SRP"}) {
  615. my $add_httptls;
  616. for(@protocols) {
  617. if($_ =~ /^https(-ipv6|)$/) {
  618. $add_httptls=1;
  619. last;
  620. }
  621. }
  622. if($add_httptls && (! grep /^httptls$/, @protocols)) {
  623. push @protocols, 'httptls';
  624. push @protocols, 'httptls-ipv6';
  625. }
  626. }
  627. }
  628. if(!$curl) {
  629. logmsg "unable to get curl's version, further details are:\n";
  630. logmsg "issued command: \n";
  631. logmsg "$versioncmd \n";
  632. if ($versretval == -1) {
  633. logmsg "command failed with: \n";
  634. logmsg "$versnoexec \n";
  635. }
  636. elsif ($versretval & 127) {
  637. logmsg sprintf("command died with signal %d, and %s coredump.\n",
  638. ($versretval & 127), ($versretval & 128)?"a":"no");
  639. }
  640. else {
  641. logmsg sprintf("command exited with value %d \n", $versretval >> 8);
  642. }
  643. logmsg "contents of $curlverout: \n";
  644. displaylogcontent("$curlverout");
  645. logmsg "contents of $curlvererr: \n";
  646. displaylogcontent("$curlvererr");
  647. die "couldn't get curl's version";
  648. }
  649. if(-r "../lib/curl_config.h") {
  650. open(my $conf, "<", "../lib/curl_config.h");
  651. while(<$conf>) {
  652. if($_ =~ /^\#define HAVE_GETRLIMIT/) {
  653. # set if system has getrlimit()
  654. $feature{"getrlimit"} = 1;
  655. }
  656. }
  657. close($conf);
  658. }
  659. # allow this feature only if debug mode is disabled
  660. $feature{"ld_preload"} = $feature{"ld_preload"} && !$feature{"debug"};
  661. if($feature{"ipv6"}) {
  662. # client has IPv6 support
  663. # check if the HTTP server has it!
  664. my $cmd = "server/sws".exe_ext('SRV')." --version";
  665. my @sws = `$cmd`;
  666. if($sws[0] =~ /IPv6/) {
  667. # HTTP server has IPv6 support!
  668. $http_ipv6 = 1;
  669. }
  670. # check if the FTP server has it!
  671. $cmd = "server/sockfilt".exe_ext('SRV')." --version";
  672. @sws = `$cmd`;
  673. if($sws[0] =~ /IPv6/) {
  674. # FTP server has IPv6 support!
  675. $ftp_ipv6 = 1;
  676. }
  677. }
  678. if($feature{"unix-sockets"}) {
  679. # client has Unix sockets support, check whether the HTTP server has it
  680. my $cmd = "server/sws".exe_ext('SRV')." --version";
  681. my @sws = `$cmd`;
  682. $http_unix = 1 if($sws[0] =~ /unix/);
  683. }
  684. open(my $manh, "-|", shell_quote($CURL) . " -M 2>&1");
  685. while(my $s = <$manh>) {
  686. if($s =~ /built-in manual was disabled at build-time/) {
  687. $feature{"manual"} = 0;
  688. last;
  689. }
  690. $feature{"manual"} = 1;
  691. last;
  692. }
  693. close($manh);
  694. $feature{"unittest"} = $feature{"debug"};
  695. $feature{"nghttpx"} = !!$ENV{'NGHTTPX'};
  696. $feature{"nghttpx-h3"} = !!$nghttpx_h3;
  697. #
  698. # strings that must exactly match the names used in server/disabled.c
  699. #
  700. $feature{"cookies"} = 1;
  701. # Use this as a proxy for any cryptographic authentication
  702. $feature{"crypto"} = $feature{"NTLM"} || $feature{"Kerberos"} || $feature{"SPNEGO"};
  703. $feature{"DoH"} = 1;
  704. $feature{"HTTP-auth"} = 1;
  705. $feature{"Mime"} = 1;
  706. $feature{"form-api"} = 1;
  707. $feature{"netrc"} = 1;
  708. $feature{"parsedate"} = 1;
  709. $feature{"proxy"} = 1;
  710. $feature{"shuffle-dns"} = 1;
  711. $feature{"typecheck"} = 1;
  712. $feature{"verbose-strings"} = 1;
  713. $feature{"wakeup"} = 1;
  714. $feature{"headers-api"} = 1;
  715. $feature{"xattr"} = 1;
  716. $feature{"large-time"} = 1;
  717. # make each protocol an enabled "feature"
  718. for my $p (@protocols) {
  719. $feature{$p} = 1;
  720. }
  721. # 'socks' was once here but is now removed
  722. $has_shared = `sh $CURLCONFIG --built-shared`;
  723. chomp $has_shared;
  724. $has_shared = $has_shared eq "yes";
  725. if(!$feature{"TrackMemory"} && $torture) {
  726. die "can't run torture tests since curl was built without ".
  727. "TrackMemory feature (--enable-curldebug)";
  728. }
  729. my $hostname=join(' ', runclientoutput("hostname"));
  730. my $hosttype=join(' ', runclientoutput("uname -a"));
  731. my $hostos=$^O;
  732. # display summary information about curl and the test host
  733. logmsg ("********* System characteristics ******** \n",
  734. "* $curl\n",
  735. "* $libcurl\n",
  736. "* Features: $feat\n",
  737. "* Disabled: $dis\n",
  738. "* Host: $hostname",
  739. "* System: $hosttype",
  740. "* OS: $hostos\n");
  741. if($jobs) {
  742. # Only show if not the default for now
  743. logmsg "* Jobs: $jobs\n";
  744. }
  745. if($feature{"TrackMemory"} && $feature{"threaded-resolver"}) {
  746. logmsg("*\n",
  747. "*** DISABLES memory tracking when using threaded resolver\n",
  748. "*\n");
  749. }
  750. logmsg sprintf("* Env: %s%s%s", $valgrind?"Valgrind ":"",
  751. $run_event_based?"event-based ":"",
  752. $nghttpx_h3);
  753. logmsg sprintf("%s\n", $libtool?"Libtool ":"");
  754. logmsg ("* Seed: $randseed\n");
  755. # Disable memory tracking when using threaded resolver
  756. $feature{"TrackMemory"} = $feature{"TrackMemory"} && !$feature{"threaded-resolver"};
  757. # toggle off the features that were disabled in the build
  758. for my $d(@disabled) {
  759. $feature{$d} = 0;
  760. }
  761. }
  762. #######################################################################
  763. # display information about server features
  764. #
  765. sub displayserverfeatures {
  766. logmsg sprintf("* Servers: %s", $stunnel?"SSL ":"");
  767. logmsg sprintf("%s", $http_ipv6?"HTTP-IPv6 ":"");
  768. logmsg sprintf("%s", $http_unix?"HTTP-unix ":"");
  769. logmsg sprintf("%s\n", $ftp_ipv6?"FTP-IPv6 ":"");
  770. logmsg "***************************************** \n";
  771. }
  772. #######################################################################
  773. # Provide time stamps for single test skipped events
  774. #
  775. sub timestampskippedevents {
  776. my $testnum = $_[0];
  777. return if((not defined($testnum)) || ($testnum < 1));
  778. if($timestats) {
  779. if($timevrfyend{$testnum}) {
  780. return;
  781. }
  782. elsif($timesrvrlog{$testnum}) {
  783. $timevrfyend{$testnum} = $timesrvrlog{$testnum};
  784. return;
  785. }
  786. elsif($timetoolend{$testnum}) {
  787. $timevrfyend{$testnum} = $timetoolend{$testnum};
  788. $timesrvrlog{$testnum} = $timetoolend{$testnum};
  789. }
  790. elsif($timetoolini{$testnum}) {
  791. $timevrfyend{$testnum} = $timetoolini{$testnum};
  792. $timesrvrlog{$testnum} = $timetoolini{$testnum};
  793. $timetoolend{$testnum} = $timetoolini{$testnum};
  794. }
  795. elsif($timesrvrend{$testnum}) {
  796. $timevrfyend{$testnum} = $timesrvrend{$testnum};
  797. $timesrvrlog{$testnum} = $timesrvrend{$testnum};
  798. $timetoolend{$testnum} = $timesrvrend{$testnum};
  799. $timetoolini{$testnum} = $timesrvrend{$testnum};
  800. }
  801. elsif($timesrvrini{$testnum}) {
  802. $timevrfyend{$testnum} = $timesrvrini{$testnum};
  803. $timesrvrlog{$testnum} = $timesrvrini{$testnum};
  804. $timetoolend{$testnum} = $timesrvrini{$testnum};
  805. $timetoolini{$testnum} = $timesrvrini{$testnum};
  806. $timesrvrend{$testnum} = $timesrvrini{$testnum};
  807. }
  808. elsif($timeprepini{$testnum}) {
  809. $timevrfyend{$testnum} = $timeprepini{$testnum};
  810. $timesrvrlog{$testnum} = $timeprepini{$testnum};
  811. $timetoolend{$testnum} = $timeprepini{$testnum};
  812. $timetoolini{$testnum} = $timeprepini{$testnum};
  813. $timesrvrend{$testnum} = $timeprepini{$testnum};
  814. $timesrvrini{$testnum} = $timeprepini{$testnum};
  815. }
  816. }
  817. }
  818. # Setup CI Test Run
  819. sub citest_starttestrun {
  820. if(azure_check_environment()) {
  821. $AZURE_RUN_ID = azure_create_test_run($ACURL);
  822. logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose);
  823. }
  824. # Appveyor doesn't require anything here
  825. }
  826. # Register the test case with the CI runner
  827. sub citest_starttest {
  828. my $testnum = $_[0];
  829. # get the name of the test early
  830. my $testname= (getpart("client", "name"))[0];
  831. chomp $testname;
  832. # create test result in CI services
  833. if(azure_check_environment() && $AZURE_RUN_ID) {
  834. $AZURE_RESULT_ID = azure_create_test_result($ACURL, $AZURE_RUN_ID, $testnum, $testname);
  835. }
  836. elsif(appveyor_check_environment()) {
  837. appveyor_create_test_result($ACURL, $testnum, $testname);
  838. }
  839. }
  840. # Submit the test case result with the CI runner
  841. sub citest_finishtest {
  842. my ($testnum, $error) = @_;
  843. # update test result in CI services
  844. if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
  845. $AZURE_RESULT_ID = azure_update_test_result($ACURL, $AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
  846. $timeprepini{$testnum}, $timevrfyend{$testnum});
  847. }
  848. elsif(appveyor_check_environment()) {
  849. appveyor_update_test_result($ACURL, $testnum, $error, $timeprepini{$testnum}, $timevrfyend{$testnum});
  850. }
  851. }
  852. # Complete CI test run
  853. sub citest_finishtestrun {
  854. if(azure_check_environment() && $AZURE_RUN_ID) {
  855. $AZURE_RUN_ID = azure_update_test_run($ACURL, $AZURE_RUN_ID);
  856. }
  857. # Appveyor doesn't require anything here
  858. }
  859. # add one set of test timings from the runner to global set
  860. sub updatetesttimings {
  861. my ($testnum, %testtimings)=@_;
  862. if(defined $testtimings{"timeprepini"}) {
  863. $timeprepini{$testnum} = $testtimings{"timeprepini"};
  864. }
  865. if(defined $testtimings{"timesrvrini"}) {
  866. $timesrvrini{$testnum} = $testtimings{"timesrvrini"};
  867. }
  868. if(defined $testtimings{"timesrvrend"}) {
  869. $timesrvrend{$testnum} = $testtimings{"timesrvrend"};
  870. }
  871. if(defined $testtimings{"timetoolini"}) {
  872. $timetoolini{$testnum} = $testtimings{"timetoolini"};
  873. }
  874. if(defined $testtimings{"timetoolend"}) {
  875. $timetoolend{$testnum} = $testtimings{"timetoolend"};
  876. }
  877. if(defined $testtimings{"timesrvrlog"}) {
  878. $timesrvrlog{$testnum} = $testtimings{"timesrvrlog"};
  879. }
  880. }
  881. #######################################################################
  882. # Return the log directory for the given test runner
  883. sub getrunnernumlogdir {
  884. my $runnernum = $_[0];
  885. return $jobs > 1 ? "$LOGDIR/$runnernum" : $LOGDIR;
  886. }
  887. #######################################################################
  888. # Return the log directory for the given test runner ID
  889. sub getrunnerlogdir {
  890. my $runnerid = $_[0];
  891. if($jobs <= 1) {
  892. return $LOGDIR;
  893. }
  894. # TODO: speed up this O(n) operation
  895. for my $runnernum (keys %runnerids) {
  896. if($runnerid eq $runnerids{$runnernum}) {
  897. return "$LOGDIR/$runnernum";
  898. }
  899. }
  900. die "Internal error: runner ID $runnerid not found";
  901. }
  902. #######################################################################
  903. # Verify that this test case should be run
  904. sub singletest_shouldrun {
  905. my $testnum = $_[0];
  906. my $why; # why the test won't be run
  907. my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
  908. my @what; # what features are needed
  909. if($disttests !~ /test$testnum(\W|\z)/ ) {
  910. logmsg "Warning: test$testnum not present in tests/data/Makefile.inc\n";
  911. }
  912. if($disabled{$testnum}) {
  913. if(!$run_disabled) {
  914. $why = "listed in DISABLED";
  915. }
  916. else {
  917. logmsg "Warning: test$testnum is explicitly disabled\n";
  918. }
  919. }
  920. if($ignored{$testnum}) {
  921. logmsg "Warning: test$testnum result is ignored\n";
  922. $errorreturncode = 2;
  923. }
  924. if(loadtest("${TESTDIR}/test${testnum}")) {
  925. if($verbose) {
  926. # this is not a test
  927. logmsg "RUN: $testnum doesn't look like a test case\n";
  928. }
  929. $why = "no test";
  930. }
  931. else {
  932. @what = getpart("client", "features");
  933. }
  934. # We require a feature to be present
  935. for(@what) {
  936. my $f = $_;
  937. $f =~ s/\s//g;
  938. if($f =~ /^([^!].*)$/) {
  939. if($feature{$1}) {
  940. next;
  941. }
  942. $why = "curl lacks $1 support";
  943. last;
  944. }
  945. }
  946. # We require a feature to not be present
  947. if(!$why) {
  948. for(@what) {
  949. my $f = $_;
  950. $f =~ s/\s//g;
  951. if($f =~ /^!(.*)$/) {
  952. if(!$feature{$1}) {
  953. next;
  954. }
  955. }
  956. else {
  957. next;
  958. }
  959. $why = "curl has $1 support";
  960. last;
  961. }
  962. }
  963. my @info_keywords;
  964. if(!$why) {
  965. @info_keywords = getpart("info", "keywords");
  966. if(!$info_keywords[0]) {
  967. $why = "missing the <keywords> section!";
  968. }
  969. my $match;
  970. for my $k (@info_keywords) {
  971. chomp $k;
  972. if ($disabled_keywords{lc($k)}) {
  973. $why = "disabled by keyword";
  974. }
  975. elsif ($enabled_keywords{lc($k)}) {
  976. $match = 1;
  977. }
  978. if ($ignored_keywords{lc($k)}) {
  979. logmsg "Warning: test$testnum result is ignored due to $k\n";
  980. $errorreturncode = 2;
  981. }
  982. }
  983. if(!$why && !$match && %enabled_keywords) {
  984. $why = "disabled by missing keyword";
  985. }
  986. }
  987. if (!$why && defined $custom_skip_reasons{test}{$testnum}) {
  988. $why = $custom_skip_reasons{test}{$testnum};
  989. }
  990. if (!$why && defined $custom_skip_reasons{tool}) {
  991. foreach my $tool (getpart("client", "tool")) {
  992. foreach my $tool_skip_pattern (keys %{$custom_skip_reasons{tool}}) {
  993. if ($tool =~ /$tool_skip_pattern/i) {
  994. $why = $custom_skip_reasons{tool}{$tool_skip_pattern};
  995. }
  996. }
  997. }
  998. }
  999. if (!$why && defined $custom_skip_reasons{keyword}) {
  1000. foreach my $keyword (@info_keywords) {
  1001. foreach my $keyword_skip_pattern (keys %{$custom_skip_reasons{keyword}}) {
  1002. if ($keyword =~ /$keyword_skip_pattern/i) {
  1003. $why = $custom_skip_reasons{keyword}{$keyword_skip_pattern};
  1004. }
  1005. }
  1006. }
  1007. }
  1008. return ($why, $errorreturncode);
  1009. }
  1010. #######################################################################
  1011. # Print the test name and count tests
  1012. sub singletest_count {
  1013. my ($testnum, $why) = @_;
  1014. if($why && !$listonly) {
  1015. # there's a problem, count it as "skipped"
  1016. $skipped{$why}++;
  1017. $teststat[$testnum]=$why; # store reason for this test case
  1018. if(!$short) {
  1019. if($skipped{$why} <= 3) {
  1020. # show only the first three skips for each reason
  1021. logmsg sprintf("test %04d SKIPPED: $why\n", $testnum);
  1022. }
  1023. }
  1024. timestampskippedevents($testnum);
  1025. return -1;
  1026. }
  1027. # At this point we've committed to run this test
  1028. logmsg sprintf("test %04d...", $testnum) if(!$automakestyle);
  1029. # name of the test
  1030. my $testname= (getpart("client", "name"))[0];
  1031. chomp $testname;
  1032. logmsg "[$testname]\n" if(!$short);
  1033. if($listonly) {
  1034. timestampskippedevents($testnum);
  1035. }
  1036. return 0;
  1037. }
  1038. #######################################################################
  1039. # Verify test succeeded
  1040. sub singletest_check {
  1041. my ($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind)=@_;
  1042. # Skip all the verification on torture tests
  1043. if ($torture) {
  1044. # timestamp test result verification end
  1045. $timevrfyend{$testnum} = Time::HiRes::time();
  1046. return -2;
  1047. }
  1048. my $logdir = getrunnerlogdir($runnerid);
  1049. my @err = getpart("verify", "errorcode");
  1050. my $errorcode = $err[0] || "0";
  1051. my $ok="";
  1052. my $res;
  1053. chomp $errorcode;
  1054. my $testname= (getpart("client", "name"))[0];
  1055. chomp $testname;
  1056. # what parts to cut off from stdout/stderr
  1057. my @stripfile = getpart("verify", "stripfile");
  1058. my @validstdout = getpart("verify", "stdout");
  1059. # get all attributes
  1060. my %hash = getpartattr("verify", "stdout");
  1061. my $loadfile = $hash{'loadfile'};
  1062. if ($loadfile) {
  1063. open(my $tmp, "<", "$loadfile") || die "Cannot open file $loadfile: $!";
  1064. @validstdout = <$tmp>;
  1065. close($tmp);
  1066. # Enforce LF newlines on load
  1067. s/\r\n/\n/g for @validstdout;
  1068. }
  1069. if (@validstdout) {
  1070. # verify redirected stdout
  1071. my @actual = loadarray(stdoutfilename($logdir, $testnum));
  1072. foreach my $strip (@stripfile) {
  1073. chomp $strip;
  1074. my @newgen;
  1075. for(@actual) {
  1076. eval $strip;
  1077. if($_) {
  1078. push @newgen, $_;
  1079. }
  1080. }
  1081. # this is to get rid of array entries that vanished (zero
  1082. # length) because of replacements
  1083. @actual = @newgen;
  1084. }
  1085. # get the mode attribute
  1086. my $filemode=$hash{'mode'};
  1087. if($filemode && ($filemode eq "text") && $has_textaware) {
  1088. # text mode when running on windows: fix line endings
  1089. s/\r\n/\n/g for @validstdout;
  1090. s/\n/\r\n/g for @validstdout;
  1091. }
  1092. if($hash{'nonewline'}) {
  1093. # Yes, we must cut off the final newline from the final line
  1094. # of the protocol data
  1095. chomp($validstdout[-1]);
  1096. }
  1097. if($hash{'crlf'} ||
  1098. ($feature{"hyper"} && ($keywords{"HTTP"}
  1099. || $keywords{"HTTPS"}))) {
  1100. subnewlines(0, \$_) for @validstdout;
  1101. }
  1102. $res = compare($runnerid, $testnum, $testname, "stdout", \@actual, \@validstdout);
  1103. if($res) {
  1104. return -1;
  1105. }
  1106. $ok .= "s";
  1107. }
  1108. else {
  1109. $ok .= "-"; # stdout not checked
  1110. }
  1111. my @validstderr = getpart("verify", "stderr");
  1112. if (@validstderr) {
  1113. # verify redirected stderr
  1114. my @actual = loadarray(stderrfilename($logdir, $testnum));
  1115. foreach my $strip (@stripfile) {
  1116. chomp $strip;
  1117. my @newgen;
  1118. for(@actual) {
  1119. eval $strip;
  1120. if($_) {
  1121. push @newgen, $_;
  1122. }
  1123. }
  1124. # this is to get rid of array entries that vanished (zero
  1125. # length) because of replacements
  1126. @actual = @newgen;
  1127. }
  1128. # get all attributes
  1129. my %hash = getpartattr("verify", "stderr");
  1130. # get the mode attribute
  1131. my $filemode=$hash{'mode'};
  1132. if($filemode && ($filemode eq "text") && $feature{"hyper"}) {
  1133. # text mode check in hyper-mode. Sometimes necessary if the stderr
  1134. # data *looks* like HTTP and thus has gotten CRLF newlines
  1135. # mistakenly
  1136. s/\r\n/\n/g for @validstderr;
  1137. }
  1138. if($filemode && ($filemode eq "text") && $has_textaware) {
  1139. # text mode when running on windows: fix line endings
  1140. s/\r\n/\n/g for @validstderr;
  1141. s/\n/\r\n/g for @validstderr;
  1142. }
  1143. if($hash{'nonewline'}) {
  1144. # Yes, we must cut off the final newline from the final line
  1145. # of the protocol data
  1146. chomp($validstderr[-1]);
  1147. }
  1148. $res = compare($runnerid, $testnum, $testname, "stderr", \@actual, \@validstderr);
  1149. if($res) {
  1150. return -1;
  1151. }
  1152. $ok .= "r";
  1153. }
  1154. else {
  1155. $ok .= "-"; # stderr not checked
  1156. }
  1157. # what to cut off from the live protocol sent by curl
  1158. my @strip = getpart("verify", "strip");
  1159. # what parts to cut off from the protocol & upload
  1160. my @strippart = getpart("verify", "strippart");
  1161. # this is the valid protocol blurb curl should generate
  1162. my @protocol= getpart("verify", "protocol");
  1163. if(@protocol) {
  1164. # Verify the sent request
  1165. my @out = loadarray("$logdir/$SERVERIN");
  1166. # check if there's any attributes on the verify/protocol section
  1167. my %hash = getpartattr("verify", "protocol");
  1168. if($hash{'nonewline'}) {
  1169. # Yes, we must cut off the final newline from the final line
  1170. # of the protocol data
  1171. chomp($protocol[-1]);
  1172. }
  1173. for(@strip) {
  1174. # strip off all lines that match the patterns from both arrays
  1175. chomp $_;
  1176. @out = striparray( $_, \@out);
  1177. @protocol= striparray( $_, \@protocol);
  1178. }
  1179. for my $strip (@strippart) {
  1180. chomp $strip;
  1181. for(@out) {
  1182. eval $strip;
  1183. }
  1184. }
  1185. if($hash{'crlf'}) {
  1186. subnewlines(1, \$_) for @protocol;
  1187. }
  1188. if((!$out[0] || ($out[0] eq "")) && $protocol[0]) {
  1189. logmsg "\n $testnum: protocol FAILED!\n".
  1190. " There was no content at all in the file $logdir/$SERVERIN.\n".
  1191. " Server glitch? Total curl failure? Returned: $cmdres\n";
  1192. # timestamp test result verification end
  1193. $timevrfyend{$testnum} = Time::HiRes::time();
  1194. return -1;
  1195. }
  1196. $res = compare($runnerid, $testnum, $testname, "protocol", \@out, \@protocol);
  1197. if($res) {
  1198. return -1;
  1199. }
  1200. $ok .= "p";
  1201. }
  1202. else {
  1203. $ok .= "-"; # protocol not checked
  1204. }
  1205. my %replyattr = getpartattr("reply", "data");
  1206. my @reply;
  1207. if (partexists("reply", "datacheck")) {
  1208. for my $partsuffix (('', '1', '2', '3', '4')) {
  1209. my @replycheckpart = getpart("reply", "datacheck".$partsuffix);
  1210. if(@replycheckpart) {
  1211. my %replycheckpartattr = getpartattr("reply", "datacheck".$partsuffix);
  1212. # get the mode attribute
  1213. my $filemode=$replycheckpartattr{'mode'};
  1214. if($filemode && ($filemode eq "text") && $has_textaware) {
  1215. # text mode when running on windows: fix line endings
  1216. s/\r\n/\n/g for @replycheckpart;
  1217. s/\n/\r\n/g for @replycheckpart;
  1218. }
  1219. if($replycheckpartattr{'nonewline'}) {
  1220. # Yes, we must cut off the final newline from the final line
  1221. # of the datacheck
  1222. chomp($replycheckpart[-1]);
  1223. }
  1224. if($replycheckpartattr{'crlf'} ||
  1225. ($feature{"hyper"} && ($keywords{"HTTP"}
  1226. || $keywords{"HTTPS"}))) {
  1227. subnewlines(0, \$_) for @replycheckpart;
  1228. }
  1229. push(@reply, @replycheckpart);
  1230. }
  1231. }
  1232. }
  1233. else {
  1234. # check against the data section
  1235. @reply = getpart("reply", "data");
  1236. if(@reply) {
  1237. if($replyattr{'nonewline'}) {
  1238. # cut off the final newline from the final line of the data
  1239. chomp($reply[-1]);
  1240. }
  1241. }
  1242. # get the mode attribute
  1243. my $filemode=$replyattr{'mode'};
  1244. if($filemode && ($filemode eq "text") && $has_textaware) {
  1245. # text mode when running on windows: fix line endings
  1246. s/\r\n/\n/g for @reply;
  1247. s/\n/\r\n/g for @reply;
  1248. }
  1249. if($replyattr{'crlf'} ||
  1250. ($feature{"hyper"} && ($keywords{"HTTP"}
  1251. || $keywords{"HTTPS"}))) {
  1252. subnewlines(0, \$_) for @reply;
  1253. }
  1254. }
  1255. if(!$replyattr{'nocheck'} && (@reply || $replyattr{'sendzero'})) {
  1256. # verify the received data
  1257. my @out = loadarray($CURLOUT);
  1258. $res = compare($runnerid, $testnum, $testname, "data", \@out, \@reply);
  1259. if ($res) {
  1260. return -1;
  1261. }
  1262. $ok .= "d";
  1263. }
  1264. else {
  1265. $ok .= "-"; # data not checked
  1266. }
  1267. # if this section exists, we verify upload
  1268. my @upload = getpart("verify", "upload");
  1269. if(@upload) {
  1270. my %hash = getpartattr("verify", "upload");
  1271. if($hash{'nonewline'}) {
  1272. # cut off the final newline from the final line of the upload data
  1273. chomp($upload[-1]);
  1274. }
  1275. # verify uploaded data
  1276. my @out = loadarray("$logdir/upload.$testnum");
  1277. for my $strip (@strippart) {
  1278. chomp $strip;
  1279. for(@out) {
  1280. eval $strip;
  1281. }
  1282. }
  1283. $res = compare($runnerid, $testnum, $testname, "upload", \@out, \@upload);
  1284. if ($res) {
  1285. return -1;
  1286. }
  1287. $ok .= "u";
  1288. }
  1289. else {
  1290. $ok .= "-"; # upload not checked
  1291. }
  1292. # this is the valid protocol blurb curl should generate to a proxy
  1293. my @proxyprot = getpart("verify", "proxy");
  1294. if(@proxyprot) {
  1295. # Verify the sent proxy request
  1296. # check if there's any attributes on the verify/protocol section
  1297. my %hash = getpartattr("verify", "proxy");
  1298. if($hash{'nonewline'}) {
  1299. # Yes, we must cut off the final newline from the final line
  1300. # of the protocol data
  1301. chomp($proxyprot[-1]);
  1302. }
  1303. my @out = loadarray("$logdir/$PROXYIN");
  1304. for(@strip) {
  1305. # strip off all lines that match the patterns from both arrays
  1306. chomp $_;
  1307. @out = striparray( $_, \@out);
  1308. @proxyprot= striparray( $_, \@proxyprot);
  1309. }
  1310. for my $strip (@strippart) {
  1311. chomp $strip;
  1312. for(@out) {
  1313. eval $strip;
  1314. }
  1315. }
  1316. if($hash{'crlf'} ||
  1317. ($feature{"hyper"} && ($keywords{"HTTP"} || $keywords{"HTTPS"}))) {
  1318. subnewlines(0, \$_) for @proxyprot;
  1319. }
  1320. $res = compare($runnerid, $testnum, $testname, "proxy", \@out, \@proxyprot);
  1321. if($res) {
  1322. return -1;
  1323. }
  1324. $ok .= "P";
  1325. }
  1326. else {
  1327. $ok .= "-"; # protocol not checked
  1328. }
  1329. my $outputok;
  1330. for my $partsuffix (('', '1', '2', '3', '4')) {
  1331. my @outfile=getpart("verify", "file".$partsuffix);
  1332. if(@outfile || partexists("verify", "file".$partsuffix) ) {
  1333. # we're supposed to verify a dynamically generated file!
  1334. my %hash = getpartattr("verify", "file".$partsuffix);
  1335. my $filename=$hash{'name'};
  1336. if(!$filename) {
  1337. logmsg " $testnum: IGNORED: section verify=>file$partsuffix ".
  1338. "has no name attribute\n";
  1339. if (runnerac_stopservers($runnerid)) {
  1340. logmsg "ERROR: runner $runnerid seems to have died\n";
  1341. } else {
  1342. # TODO: this is a blocking call that will stall the controller,
  1343. if($verbose) {
  1344. logmsg "WARNING: blocking call in async function\n";
  1345. }
  1346. # but this error condition should never happen except during
  1347. # development.
  1348. my ($rid, $unexpected, $logs) = runnerar($runnerid);
  1349. if(!$rid) {
  1350. logmsg "ERROR: runner $runnerid seems to have died\n";
  1351. } else {
  1352. logmsg $logs;
  1353. }
  1354. }
  1355. # timestamp test result verification end
  1356. $timevrfyend{$testnum} = Time::HiRes::time();
  1357. return -1;
  1358. }
  1359. my @generated=loadarray($filename);
  1360. # what parts to cut off from the file
  1361. my @stripfilepar = getpart("verify", "stripfile".$partsuffix);
  1362. my $filemode=$hash{'mode'};
  1363. if($filemode && ($filemode eq "text") && $has_textaware) {
  1364. # text mode when running on windows: fix line endings
  1365. s/\r\n/\n/g for @outfile;
  1366. s/\n/\r\n/g for @outfile;
  1367. }
  1368. if($hash{'crlf'} ||
  1369. ($feature{"hyper"} && ($keywords{"HTTP"}
  1370. || $keywords{"HTTPS"}))) {
  1371. subnewlines(0, \$_) for @outfile;
  1372. }
  1373. for my $strip (@stripfilepar) {
  1374. chomp $strip;
  1375. my @newgen;
  1376. for(@generated) {
  1377. eval $strip;
  1378. if($_) {
  1379. push @newgen, $_;
  1380. }
  1381. }
  1382. # this is to get rid of array entries that vanished (zero
  1383. # length) because of replacements
  1384. @generated = @newgen;
  1385. }
  1386. if($hash{'nonewline'}) {
  1387. # cut off the final newline from the final line of the
  1388. # output data
  1389. chomp($outfile[-1]);
  1390. }
  1391. $res = compare($runnerid, $testnum, $testname, "output ($filename)",
  1392. \@generated, \@outfile);
  1393. if($res) {
  1394. return -1;
  1395. }
  1396. $outputok = 1; # output checked
  1397. }
  1398. }
  1399. $ok .= ($outputok) ? "o" : "-"; # output checked or not
  1400. # verify SOCKS proxy details
  1401. my @socksprot = getpart("verify", "socks");
  1402. if(@socksprot) {
  1403. # Verify the sent SOCKS proxy details
  1404. my @out = loadarray("$logdir/$SOCKSIN");
  1405. $res = compare($runnerid, $testnum, $testname, "socks", \@out, \@socksprot);
  1406. if($res) {
  1407. return -1;
  1408. }
  1409. }
  1410. # accept multiple comma-separated error codes
  1411. my @splerr = split(/ *, */, $errorcode);
  1412. my $errok;
  1413. foreach my $e (@splerr) {
  1414. if($e == $cmdres) {
  1415. # a fine error code
  1416. $errok = 1;
  1417. last;
  1418. }
  1419. }
  1420. if($errok) {
  1421. $ok .= "e";
  1422. }
  1423. else {
  1424. if(!$short) {
  1425. logmsg sprintf("\n%s returned $cmdres, when expecting %s\n",
  1426. (!$tool)?"curl":$tool, $errorcode);
  1427. }
  1428. logmsg " $testnum: exit FAILED\n";
  1429. # timestamp test result verification end
  1430. $timevrfyend{$testnum} = Time::HiRes::time();
  1431. return -1;
  1432. }
  1433. if($feature{"TrackMemory"}) {
  1434. if(! -f "$logdir/$MEMDUMP") {
  1435. my %cmdhash = getpartattr("client", "command");
  1436. my $cmdtype = $cmdhash{'type'} || "default";
  1437. logmsg "\n** ALERT! memory tracking with no output file?\n"
  1438. if(!$cmdtype eq "perl");
  1439. $ok .= "-"; # problem with memory checking
  1440. }
  1441. else {
  1442. my @memdata=`$memanalyze "$logdir/$MEMDUMP"`;
  1443. my $leak=0;
  1444. for(@memdata) {
  1445. if($_ ne "") {
  1446. # well it could be other memory problems as well, but
  1447. # we call it leak for short here
  1448. $leak=1;
  1449. }
  1450. }
  1451. if($leak) {
  1452. logmsg "\n** MEMORY FAILURE\n";
  1453. logmsg @memdata;
  1454. # timestamp test result verification end
  1455. $timevrfyend{$testnum} = Time::HiRes::time();
  1456. return -1;
  1457. }
  1458. else {
  1459. $ok .= "m";
  1460. }
  1461. }
  1462. }
  1463. else {
  1464. $ok .= "-"; # memory not checked
  1465. }
  1466. if($valgrind) {
  1467. if($usedvalgrind) {
  1468. if(!opendir(DIR, "$logdir")) {
  1469. logmsg "ERROR: unable to read $logdir\n";
  1470. # timestamp test result verification end
  1471. $timevrfyend{$testnum} = Time::HiRes::time();
  1472. return -1;
  1473. }
  1474. my @files = readdir(DIR);
  1475. closedir(DIR);
  1476. my $vgfile;
  1477. foreach my $file (@files) {
  1478. if($file =~ /^valgrind$testnum(\..*|)$/) {
  1479. $vgfile = $file;
  1480. last;
  1481. }
  1482. }
  1483. if(!$vgfile) {
  1484. logmsg "ERROR: valgrind log file missing for test $testnum\n";
  1485. # timestamp test result verification end
  1486. $timevrfyend{$testnum} = Time::HiRes::time();
  1487. return -1;
  1488. }
  1489. my @e = valgrindparse("$logdir/$vgfile");
  1490. if(@e && $e[0]) {
  1491. if($automakestyle) {
  1492. logmsg "FAIL: $testnum - $testname - valgrind\n";
  1493. }
  1494. else {
  1495. logmsg " valgrind ERROR ";
  1496. logmsg @e;
  1497. }
  1498. # timestamp test result verification end
  1499. $timevrfyend{$testnum} = Time::HiRes::time();
  1500. return -1;
  1501. }
  1502. $ok .= "v";
  1503. }
  1504. else {
  1505. if($verbose) {
  1506. logmsg " valgrind SKIPPED\n";
  1507. }
  1508. $ok .= "-"; # skipped
  1509. }
  1510. }
  1511. else {
  1512. $ok .= "-"; # valgrind not checked
  1513. }
  1514. # add 'E' for event-based
  1515. $ok .= $run_event_based ? "E" : "-";
  1516. logmsg "$ok " if(!$short);
  1517. # timestamp test result verification end
  1518. $timevrfyend{$testnum} = Time::HiRes::time();
  1519. return 0;
  1520. }
  1521. #######################################################################
  1522. # Report a successful test
  1523. sub singletest_success {
  1524. my ($testnum, $count, $total, $errorreturncode)=@_;
  1525. my $sofar= time()-$start;
  1526. my $esttotal = $sofar/$count * $total;
  1527. my $estleft = $esttotal - $sofar;
  1528. my $timeleft=sprintf("remaining: %02d:%02d",
  1529. $estleft/60,
  1530. $estleft%60);
  1531. my $took = $timevrfyend{$testnum} - $timeprepini{$testnum};
  1532. my $duration = sprintf("duration: %02d:%02d",
  1533. $sofar/60, $sofar%60);
  1534. if(!$automakestyle) {
  1535. logmsg sprintf("OK (%-3d out of %-3d, %s, took %.3fs, %s)\n",
  1536. $count, $total, $timeleft, $took, $duration);
  1537. }
  1538. else {
  1539. my $testname= (getpart("client", "name"))[0];
  1540. chomp $testname;
  1541. logmsg "PASS: $testnum - $testname\n";
  1542. }
  1543. if($errorreturncode==2) {
  1544. logmsg "Warning: test$testnum result is ignored, but passed!\n";
  1545. }
  1546. }
  1547. #######################################################################
  1548. # Run a single specified test case
  1549. # This is structured as a state machine which changes state after an
  1550. # asynchronous call is made that awaits a response. The function returns with
  1551. # an error code and a flag that indicates if the state machine has completed,
  1552. # which means (if not) the function must be called again once the response has
  1553. # arrived.
  1554. #
  1555. sub singletest {
  1556. my ($runnerid, $testnum, $count, $total)=@_;
  1557. # start buffering logmsg; stop it on return
  1558. logmsg_bufferfortest($runnerid);
  1559. if(!exists $singletest_state{$runnerid}) {
  1560. # First time in singletest() for this test
  1561. $singletest_state{$runnerid} = ST_INIT;
  1562. }
  1563. if($singletest_state{$runnerid} == ST_INIT) {
  1564. my $logdir = getrunnerlogdir($runnerid);
  1565. # first, remove all lingering log & lock files
  1566. if((!cleardir($logdir) || !cleardir("$logdir/$LOCKDIR"))
  1567. && $clearlocks) {
  1568. # On Windows, lock files can't be deleted when the process still
  1569. # has them open, so kill those processes first
  1570. if(runnerac_clearlocks($runnerid, "$logdir/$LOCKDIR")) {
  1571. logmsg "ERROR: runner $runnerid seems to have died\n";
  1572. $singletest_state{$runnerid} = ST_INIT;
  1573. return (-1, 0);
  1574. }
  1575. $singletest_state{$runnerid} = ST_CLEARLOCKS;
  1576. } else {
  1577. $singletest_state{$runnerid} = ST_INITED;
  1578. # Recursively call the state machine again because there is no
  1579. # event expected that would otherwise trigger a new call.
  1580. return singletest(@_);
  1581. }
  1582. } elsif($singletest_state{$runnerid} == ST_CLEARLOCKS) {
  1583. my ($rid, $logs) = runnerar($runnerid);
  1584. if(!$rid) {
  1585. logmsg "ERROR: runner $runnerid seems to have died\n";
  1586. $singletest_state{$runnerid} = ST_INIT;
  1587. return (-1, 0);
  1588. }
  1589. logmsg $logs;
  1590. my $logdir = getrunnerlogdir($runnerid);
  1591. cleardir($logdir);
  1592. $singletest_state{$runnerid} = ST_INITED;
  1593. # Recursively call the state machine again because there is no
  1594. # event expected that would otherwise trigger a new call.
  1595. return singletest(@_);
  1596. } elsif($singletest_state{$runnerid} == ST_INITED) {
  1597. ###################################################################
  1598. # Restore environment variables that were modified in a previous run.
  1599. # Test definition may instruct to (un)set environment vars.
  1600. # This is done this early so that leftover variables don't affect
  1601. # starting servers or CI registration.
  1602. # restore_test_env(1);
  1603. ###################################################################
  1604. # Load test file so CI registration can get the right data before the
  1605. # runner is called
  1606. loadtest("${TESTDIR}/test${testnum}");
  1607. ###################################################################
  1608. # Register the test case with the CI environment
  1609. citest_starttest($testnum);
  1610. if(runnerac_test_preprocess($runnerid, $testnum)) {
  1611. logmsg "ERROR: runner $runnerid seems to have died\n";
  1612. $singletest_state{$runnerid} = ST_INIT;
  1613. return (-1, 0);
  1614. }
  1615. $singletest_state{$runnerid} = ST_PREPROCESS;
  1616. } elsif($singletest_state{$runnerid} == ST_PREPROCESS) {
  1617. my ($rid, $why, $error, $logs, $testtimings) = runnerar($runnerid);
  1618. if(!$rid) {
  1619. logmsg "ERROR: runner $runnerid seems to have died\n";
  1620. $singletest_state{$runnerid} = ST_INIT;
  1621. return (-1, 0);
  1622. }
  1623. logmsg $logs;
  1624. updatetesttimings($testnum, %$testtimings);
  1625. if($error == -2) {
  1626. if($postmortem) {
  1627. # Error indicates an actual problem starting the server, so
  1628. # display the server logs
  1629. displaylogs($rid, $testnum);
  1630. }
  1631. }
  1632. #######################################################################
  1633. # Load test file for this test number
  1634. my $logdir = getrunnerlogdir($runnerid);
  1635. loadtest("${logdir}/test${testnum}");
  1636. #######################################################################
  1637. # Print the test name and count tests
  1638. $error = singletest_count($testnum, $why);
  1639. if($error) {
  1640. # Submit the test case result with the CI environment
  1641. citest_finishtest($testnum, $error);
  1642. $singletest_state{$runnerid} = ST_INIT;
  1643. logmsg singletest_dumplogs();
  1644. return ($error, 0);
  1645. }
  1646. #######################################################################
  1647. # Execute this test number
  1648. my $cmdres;
  1649. my $CURLOUT;
  1650. my $tool;
  1651. my $usedvalgrind;
  1652. if(runnerac_test_run($runnerid, $testnum)) {
  1653. logmsg "ERROR: runner $runnerid seems to have died\n";
  1654. $singletest_state{$runnerid} = ST_INIT;
  1655. return (-1, 0);
  1656. }
  1657. $singletest_state{$runnerid} = ST_RUN;
  1658. } elsif($singletest_state{$runnerid} == ST_RUN) {
  1659. my ($rid, $error, $logs, $testtimings, $cmdres, $CURLOUT, $tool, $usedvalgrind) = runnerar($runnerid);
  1660. if(!$rid) {
  1661. logmsg "ERROR: runner $runnerid seems to have died\n";
  1662. $singletest_state{$runnerid} = ST_INIT;
  1663. return (-1, 0);
  1664. }
  1665. logmsg $logs;
  1666. updatetesttimings($testnum, %$testtimings);
  1667. if($error == -1) {
  1668. # no further verification will occur
  1669. $timevrfyend{$testnum} = Time::HiRes::time();
  1670. my $err = ignoreresultcode($testnum);
  1671. # Submit the test case result with the CI environment
  1672. citest_finishtest($testnum, $err);
  1673. $singletest_state{$runnerid} = ST_INIT;
  1674. logmsg singletest_dumplogs();
  1675. # return a test failure, either to be reported or to be ignored
  1676. return ($err, 0);
  1677. }
  1678. elsif($error == -2) {
  1679. # fill in the missing timings on error
  1680. timestampskippedevents($testnum);
  1681. # Submit the test case result with the CI environment
  1682. citest_finishtest($testnum, $error);
  1683. $singletest_state{$runnerid} = ST_INIT;
  1684. logmsg singletest_dumplogs();
  1685. return ($error, 0);
  1686. }
  1687. elsif($error > 0) {
  1688. # no further verification will occur
  1689. $timevrfyend{$testnum} = Time::HiRes::time();
  1690. # Submit the test case result with the CI environment
  1691. citest_finishtest($testnum, $error);
  1692. $singletest_state{$runnerid} = ST_INIT;
  1693. logmsg singletest_dumplogs();
  1694. return ($error, 0);
  1695. }
  1696. #######################################################################
  1697. # Verify that the test succeeded
  1698. #
  1699. # Load test file for this test number
  1700. my $logdir = getrunnerlogdir($runnerid);
  1701. loadtest("${logdir}/test${testnum}");
  1702. readtestkeywords();
  1703. $error = singletest_check($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind);
  1704. if($error == -1) {
  1705. my $err = ignoreresultcode($testnum);
  1706. # Submit the test case result with the CI environment
  1707. citest_finishtest($testnum, $err);
  1708. $singletest_state{$runnerid} = ST_INIT;
  1709. logmsg singletest_dumplogs();
  1710. # return a test failure, either to be reported or to be ignored
  1711. return ($err, 0);
  1712. }
  1713. elsif($error == -2) {
  1714. # torture test; there is no verification, so the run result holds the
  1715. # test success code
  1716. # Submit the test case result with the CI environment
  1717. citest_finishtest($testnum, $cmdres);
  1718. $singletest_state{$runnerid} = ST_INIT;
  1719. logmsg singletest_dumplogs();
  1720. return ($cmdres, 0);
  1721. }
  1722. #######################################################################
  1723. # Report a successful test
  1724. singletest_success($testnum, $count, $total, ignoreresultcode($testnum));
  1725. # Submit the test case result with the CI environment
  1726. citest_finishtest($testnum, 0);
  1727. $singletest_state{$runnerid} = ST_INIT;
  1728. logmsg singletest_dumplogs();
  1729. return (0, 0); # state machine is finished
  1730. }
  1731. singletest_unbufferlogs();
  1732. return (0, 1); # state machine must be called again on event
  1733. }
  1734. #######################################################################
  1735. # runtimestats displays test-suite run time statistics
  1736. #
  1737. sub runtimestats {
  1738. my $lasttest = $_[0];
  1739. return if(not $timestats);
  1740. logmsg "\nTest suite total running time breakdown per task...\n\n";
  1741. my @timesrvr;
  1742. my @timeprep;
  1743. my @timetool;
  1744. my @timelock;
  1745. my @timevrfy;
  1746. my @timetest;
  1747. my $timesrvrtot = 0.0;
  1748. my $timepreptot = 0.0;
  1749. my $timetooltot = 0.0;
  1750. my $timelocktot = 0.0;
  1751. my $timevrfytot = 0.0;
  1752. my $timetesttot = 0.0;
  1753. my $counter;
  1754. for my $testnum (1 .. $lasttest) {
  1755. if($timesrvrini{$testnum}) {
  1756. $timesrvrtot += $timesrvrend{$testnum} - $timesrvrini{$testnum};
  1757. $timepreptot +=
  1758. (($timetoolini{$testnum} - $timeprepini{$testnum}) -
  1759. ($timesrvrend{$testnum} - $timesrvrini{$testnum}));
  1760. $timetooltot += $timetoolend{$testnum} - $timetoolini{$testnum};
  1761. $timelocktot += $timesrvrlog{$testnum} - $timetoolend{$testnum};
  1762. $timevrfytot += $timevrfyend{$testnum} - $timesrvrlog{$testnum};
  1763. $timetesttot += $timevrfyend{$testnum} - $timeprepini{$testnum};
  1764. push @timesrvr, sprintf("%06.3f %04d",
  1765. $timesrvrend{$testnum} - $timesrvrini{$testnum}, $testnum);
  1766. push @timeprep, sprintf("%06.3f %04d",
  1767. ($timetoolini{$testnum} - $timeprepini{$testnum}) -
  1768. ($timesrvrend{$testnum} - $timesrvrini{$testnum}), $testnum);
  1769. push @timetool, sprintf("%06.3f %04d",
  1770. $timetoolend{$testnum} - $timetoolini{$testnum}, $testnum);
  1771. push @timelock, sprintf("%06.3f %04d",
  1772. $timesrvrlog{$testnum} - $timetoolend{$testnum}, $testnum);
  1773. push @timevrfy, sprintf("%06.3f %04d",
  1774. $timevrfyend{$testnum} - $timesrvrlog{$testnum}, $testnum);
  1775. push @timetest, sprintf("%06.3f %04d",
  1776. $timevrfyend{$testnum} - $timeprepini{$testnum}, $testnum);
  1777. }
  1778. }
  1779. {
  1780. no warnings 'numeric';
  1781. @timesrvr = sort { $b <=> $a } @timesrvr;
  1782. @timeprep = sort { $b <=> $a } @timeprep;
  1783. @timetool = sort { $b <=> $a } @timetool;
  1784. @timelock = sort { $b <=> $a } @timelock;
  1785. @timevrfy = sort { $b <=> $a } @timevrfy;
  1786. @timetest = sort { $b <=> $a } @timetest;
  1787. }
  1788. logmsg "Spent ". sprintf("%08.3f ", $timesrvrtot) .
  1789. "seconds starting and verifying test harness servers.\n";
  1790. logmsg "Spent ". sprintf("%08.3f ", $timepreptot) .
  1791. "seconds reading definitions and doing test preparations.\n";
  1792. logmsg "Spent ". sprintf("%08.3f ", $timetooltot) .
  1793. "seconds actually running test tools.\n";
  1794. logmsg "Spent ". sprintf("%08.3f ", $timelocktot) .
  1795. "seconds awaiting server logs lock removal.\n";
  1796. logmsg "Spent ". sprintf("%08.3f ", $timevrfytot) .
  1797. "seconds verifying test results.\n";
  1798. logmsg "Spent ". sprintf("%08.3f ", $timetesttot) .
  1799. "seconds doing all of the above.\n";
  1800. $counter = 25;
  1801. logmsg "\nTest server starting and verification time per test ".
  1802. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1803. logmsg "-time- test\n";
  1804. logmsg "------ ----\n";
  1805. foreach my $txt (@timesrvr) {
  1806. last if((not $fullstats) && (not $counter--));
  1807. logmsg "$txt\n";
  1808. }
  1809. $counter = 10;
  1810. logmsg "\nTest definition reading and preparation time per test ".
  1811. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1812. logmsg "-time- test\n";
  1813. logmsg "------ ----\n";
  1814. foreach my $txt (@timeprep) {
  1815. last if((not $fullstats) && (not $counter--));
  1816. logmsg "$txt\n";
  1817. }
  1818. $counter = 25;
  1819. logmsg "\nTest tool execution time per test ".
  1820. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1821. logmsg "-time- test\n";
  1822. logmsg "------ ----\n";
  1823. foreach my $txt (@timetool) {
  1824. last if((not $fullstats) && (not $counter--));
  1825. logmsg "$txt\n";
  1826. }
  1827. $counter = 15;
  1828. logmsg "\nTest server logs lock removal time per test ".
  1829. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1830. logmsg "-time- test\n";
  1831. logmsg "------ ----\n";
  1832. foreach my $txt (@timelock) {
  1833. last if((not $fullstats) && (not $counter--));
  1834. logmsg "$txt\n";
  1835. }
  1836. $counter = 10;
  1837. logmsg "\nTest results verification time per test ".
  1838. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1839. logmsg "-time- test\n";
  1840. logmsg "------ ----\n";
  1841. foreach my $txt (@timevrfy) {
  1842. last if((not $fullstats) && (not $counter--));
  1843. logmsg "$txt\n";
  1844. }
  1845. $counter = 50;
  1846. logmsg "\nTotal time per test ".
  1847. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1848. logmsg "-time- test\n";
  1849. logmsg "------ ----\n";
  1850. foreach my $txt (@timetest) {
  1851. last if((not $fullstats) && (not $counter--));
  1852. logmsg "$txt\n";
  1853. }
  1854. logmsg "\n";
  1855. }
  1856. #######################################################################
  1857. # returns code indicating why a test was skipped
  1858. # 0=unknown test, 1=use test result, 2=ignore test result
  1859. #
  1860. sub ignoreresultcode {
  1861. my ($testnum)=@_;
  1862. if(defined $ignoretestcodes{$testnum}) {
  1863. return $ignoretestcodes{$testnum};
  1864. }
  1865. return 0;
  1866. }
  1867. #######################################################################
  1868. # Put the given runner ID onto the queue of runners ready for a new task
  1869. #
  1870. sub runnerready {
  1871. my ($runnerid)=@_;
  1872. push @runnersidle, $runnerid;
  1873. }
  1874. #######################################################################
  1875. # Create test runners
  1876. #
  1877. sub createrunners {
  1878. my ($numrunners)=@_;
  1879. if(! $numrunners) {
  1880. $numrunners++;
  1881. }
  1882. # create $numrunners runners with minimum 1
  1883. for my $runnernum (1..$numrunners) {
  1884. my $dir = getrunnernumlogdir($runnernum);
  1885. cleardir($dir);
  1886. mkdir($dir, 0777);
  1887. $runnerids{$runnernum} = runner_init($dir, $jobs);
  1888. runnerready($runnerids{$runnernum});
  1889. }
  1890. }
  1891. #######################################################################
  1892. # Pick a test runner for the given test
  1893. #
  1894. sub pickrunner {
  1895. my ($testnum)=@_;
  1896. scalar(@runnersidle) || die "No runners available";
  1897. return pop @runnersidle;
  1898. }
  1899. #######################################################################
  1900. # Check options to this test program
  1901. #
  1902. # Special case for CMake: replace '$TFLAGS' by the contents of the
  1903. # environment variable (if any).
  1904. if(@ARGV && $ARGV[-1] eq '$TFLAGS') {
  1905. pop @ARGV;
  1906. push(@ARGV, split(' ', $ENV{'TFLAGS'})) if defined($ENV{'TFLAGS'});
  1907. }
  1908. $valgrind = checktestcmd("valgrind");
  1909. my $number=0;
  1910. my $fromnum=-1;
  1911. my @testthis;
  1912. while(@ARGV) {
  1913. if ($ARGV[0] eq "-v") {
  1914. # verbose output
  1915. $verbose=1;
  1916. }
  1917. elsif ($ARGV[0] eq "-c") {
  1918. # use this path to curl instead of default
  1919. $DBGCURL=$CURL=$ARGV[1];
  1920. shift @ARGV;
  1921. }
  1922. elsif ($ARGV[0] eq "-vc") {
  1923. # use this path to a curl used to verify servers
  1924. # Particularly useful when you introduce a crashing bug somewhere in
  1925. # the development version as then it won't be able to run any tests
  1926. # since it can't verify the servers!
  1927. $VCURL=shell_quote($ARGV[1]);
  1928. shift @ARGV;
  1929. }
  1930. elsif ($ARGV[0] eq "-ac") {
  1931. # use this curl only to talk to APIs (currently only CI test APIs)
  1932. $ACURL=shell_quote($ARGV[1]);
  1933. shift @ARGV;
  1934. }
  1935. elsif ($ARGV[0] eq "-d") {
  1936. # have the servers display protocol output
  1937. $debugprotocol=1;
  1938. }
  1939. elsif($ARGV[0] eq "-e") {
  1940. # run the tests cases event based if possible
  1941. $run_event_based=1;
  1942. }
  1943. elsif($ARGV[0] eq "-f") {
  1944. # force - run the test case even if listed in DISABLED
  1945. $run_disabled=1;
  1946. }
  1947. elsif($ARGV[0] eq "-E") {
  1948. # load additional reasons to skip tests
  1949. shift @ARGV;
  1950. my $exclude_file = $ARGV[0];
  1951. open(my $fd, "<", $exclude_file) or die "Couldn't open '$exclude_file': $!";
  1952. while(my $line = <$fd>) {
  1953. next if ($line =~ /^#/);
  1954. chomp $line;
  1955. my ($type, $patterns, $skip_reason) = split(/\s*:\s*/, $line, 3);
  1956. die "Unsupported type: $type\n" if($type !~ /^keyword|test|tool$/);
  1957. foreach my $pattern (split(/,/, $patterns)) {
  1958. if($type eq "test") {
  1959. # Strip leading zeros in the test number
  1960. $pattern = int($pattern);
  1961. }
  1962. $custom_skip_reasons{$type}{$pattern} = $skip_reason;
  1963. }
  1964. }
  1965. close($fd);
  1966. }
  1967. elsif ($ARGV[0] eq "-g") {
  1968. # run this test with gdb
  1969. $gdbthis=1;
  1970. }
  1971. elsif ($ARGV[0] eq "-gw") {
  1972. # run this test with windowed gdb
  1973. $gdbthis=1;
  1974. $gdbxwin=1;
  1975. }
  1976. elsif($ARGV[0] eq "-s") {
  1977. # short output
  1978. $short=1;
  1979. }
  1980. elsif($ARGV[0] eq "-am") {
  1981. # automake-style output
  1982. $short=1;
  1983. $automakestyle=1;
  1984. }
  1985. elsif($ARGV[0] eq "-n") {
  1986. # no valgrind
  1987. undef $valgrind;
  1988. }
  1989. elsif($ARGV[0] eq "--no-debuginfod") {
  1990. # disable the valgrind debuginfod functionality
  1991. $no_debuginfod = 1;
  1992. }
  1993. elsif ($ARGV[0] eq "-R") {
  1994. # execute in scrambled order
  1995. $scrambleorder=1;
  1996. }
  1997. elsif($ARGV[0] =~ /^-t(.*)/) {
  1998. # torture
  1999. $torture=1;
  2000. my $xtra = $1;
  2001. if($xtra =~ s/(\d+)$//) {
  2002. $tortalloc = $1;
  2003. }
  2004. }
  2005. elsif($ARGV[0] =~ /--shallow=(\d+)/) {
  2006. # Fail no more than this amount per tests when running
  2007. # torture.
  2008. my ($num)=($1);
  2009. $shallow=$num;
  2010. }
  2011. elsif($ARGV[0] =~ /--repeat=(\d+)/) {
  2012. # Repeat-run the given tests this many times
  2013. $repeat = $1;
  2014. }
  2015. elsif($ARGV[0] =~ /--seed=(\d+)/) {
  2016. # Set a fixed random seed (used for -R and --shallow)
  2017. $randseed = $1;
  2018. }
  2019. elsif($ARGV[0] eq "-a") {
  2020. # continue anyway, even if a test fail
  2021. $anyway=1;
  2022. }
  2023. elsif($ARGV[0] eq "-o") {
  2024. shift @ARGV;
  2025. if ($ARGV[0] =~ /^(\w+)=([\w.:\/\[\]-]+)$/) {
  2026. my ($variable, $value) = ($1, $2);
  2027. eval "\$$variable='$value'" or die "Failed to set \$$variable to $value: $@";
  2028. } else {
  2029. die "Failed to parse '-o $ARGV[0]'. May contain unexpected characters.\n";
  2030. }
  2031. }
  2032. elsif($ARGV[0] eq "-p") {
  2033. $postmortem=1;
  2034. }
  2035. elsif($ARGV[0] eq "-P") {
  2036. shift @ARGV;
  2037. $proxy_address=$ARGV[0];
  2038. }
  2039. elsif($ARGV[0] eq "-L") {
  2040. # require additional library file
  2041. shift @ARGV;
  2042. require $ARGV[0];
  2043. }
  2044. elsif($ARGV[0] eq "-l") {
  2045. # lists the test case names only
  2046. $listonly=1;
  2047. }
  2048. elsif($ARGV[0] =~ /^-j(.*)/) {
  2049. # parallel jobs
  2050. $jobs=1;
  2051. my $xtra = $1;
  2052. if($xtra =~ s/(\d+)$//) {
  2053. $jobs = $1;
  2054. }
  2055. }
  2056. elsif($ARGV[0] eq "-k") {
  2057. # keep stdout and stderr files after tests
  2058. $keepoutfiles=1;
  2059. }
  2060. elsif($ARGV[0] eq "-r") {
  2061. # run time statistics needs Time::HiRes
  2062. if($Time::HiRes::VERSION) {
  2063. # presize hashes appropriately to hold an entire test run
  2064. keys(%timeprepini) = 2000;
  2065. keys(%timesrvrini) = 2000;
  2066. keys(%timesrvrend) = 2000;
  2067. keys(%timetoolini) = 2000;
  2068. keys(%timetoolend) = 2000;
  2069. keys(%timesrvrlog) = 2000;
  2070. keys(%timevrfyend) = 2000;
  2071. $timestats=1;
  2072. $fullstats=0;
  2073. }
  2074. }
  2075. elsif($ARGV[0] eq "-rf") {
  2076. # run time statistics needs Time::HiRes
  2077. if($Time::HiRes::VERSION) {
  2078. # presize hashes appropriately to hold an entire test run
  2079. keys(%timeprepini) = 2000;
  2080. keys(%timesrvrini) = 2000;
  2081. keys(%timesrvrend) = 2000;
  2082. keys(%timetoolini) = 2000;
  2083. keys(%timetoolend) = 2000;
  2084. keys(%timesrvrlog) = 2000;
  2085. keys(%timevrfyend) = 2000;
  2086. $timestats=1;
  2087. $fullstats=1;
  2088. }
  2089. }
  2090. elsif($ARGV[0] eq "-rm") {
  2091. # force removal of files by killing locking processes
  2092. $clearlocks=1;
  2093. }
  2094. elsif($ARGV[0] eq "-u") {
  2095. # error instead of warning on server unexpectedly alive
  2096. $err_unexpected=1;
  2097. }
  2098. elsif(($ARGV[0] eq "-h") || ($ARGV[0] eq "--help")) {
  2099. # show help text
  2100. print <<"EOHELP"
  2101. Usage: runtests.pl [options] [test selection(s)]
  2102. -a continue even if a test fails
  2103. -ac path use this curl only to talk to APIs (currently only CI test APIs)
  2104. -am automake style output PASS/FAIL: [number] [name]
  2105. -c path use this curl executable
  2106. -d display server debug info
  2107. -e event-based execution
  2108. -E file load the specified file to exclude certain tests
  2109. -f forcibly run even if disabled
  2110. -g run the test case with gdb
  2111. -gw run the test case with gdb as a windowed application
  2112. -h this help text
  2113. -j[N] spawn this number of processes to run tests (default 0)
  2114. -k keep stdout and stderr files present after tests
  2115. -L path require an additional perl library file to replace certain functions
  2116. -l list all test case names/descriptions
  2117. -n no valgrind
  2118. --no-debuginfod disable the valgrind debuginfod functionality
  2119. -o variable=value set internal variable to the specified value
  2120. -P proxy use the specified proxy
  2121. -p print log file contents when a test fails
  2122. -R scrambled order (uses the random seed, see --seed)
  2123. -r run time statistics
  2124. -rf full run time statistics
  2125. -rm force removal of files by killing locking processes (Windows only)
  2126. --repeat=[num] run the given tests this many times
  2127. -s short output
  2128. --seed=[num] set the random seed to a fixed number
  2129. --shallow=[num] randomly makes the torture tests "thinner"
  2130. -t[N] torture (simulate function failures); N means fail Nth function
  2131. -u error instead of warning on server unexpectedly alive
  2132. -v verbose output
  2133. -vc path use this curl only to verify the existing servers
  2134. [num] like "5 6 9" or " 5 to 22 " to run those tests only
  2135. [!num] like "!5 !6 !9" to disable those tests
  2136. [~num] like "~5 ~6 ~9" to ignore the result of those tests
  2137. [keyword] like "IPv6" to select only tests containing the key word
  2138. [!keyword] like "!cookies" to disable any tests containing the key word
  2139. [~keyword] like "~cookies" to ignore results of tests containing key word
  2140. EOHELP
  2141. ;
  2142. exit;
  2143. }
  2144. elsif($ARGV[0] =~ /^(\d+)/) {
  2145. $number = $1;
  2146. if($fromnum >= 0) {
  2147. for my $n ($fromnum .. $number) {
  2148. push @testthis, $n;
  2149. }
  2150. $fromnum = -1;
  2151. }
  2152. else {
  2153. push @testthis, $1;
  2154. }
  2155. }
  2156. elsif($ARGV[0] =~ /^to$/i) {
  2157. $fromnum = $number+1;
  2158. }
  2159. elsif($ARGV[0] =~ /^!(\d+)/) {
  2160. $fromnum = -1;
  2161. $disabled{$1}=$1;
  2162. }
  2163. elsif($ARGV[0] =~ /^~(\d+)/) {
  2164. $fromnum = -1;
  2165. $ignored{$1}=$1;
  2166. }
  2167. elsif($ARGV[0] =~ /^!(.+)/) {
  2168. $disabled_keywords{lc($1)}=$1;
  2169. }
  2170. elsif($ARGV[0] =~ /^~(.+)/) {
  2171. $ignored_keywords{lc($1)}=$1;
  2172. }
  2173. elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
  2174. $enabled_keywords{lc($1)}=$1;
  2175. }
  2176. else {
  2177. print "Unknown option: $ARGV[0]\n";
  2178. exit;
  2179. }
  2180. shift @ARGV;
  2181. }
  2182. delete $ENV{'DEBUGINFOD_URLS'} if($ENV{'DEBUGINFOD_URLS'} && $no_debuginfod);
  2183. if(!$randseed) {
  2184. my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
  2185. localtime(time);
  2186. # seed of the month. December 2019 becomes 201912
  2187. $randseed = ($year+1900)*100 + $mon+1;
  2188. print "Using curl: $CURL\n";
  2189. open(my $curlvh, "-|", shell_quote($CURL) . " --version 2>/dev/null") ||
  2190. die "could not get curl version!";
  2191. my @c = <$curlvh>;
  2192. close($curlvh) || die "could not get curl version!";
  2193. # use the first line of output and get the md5 out of it
  2194. my $str = md5($c[0]);
  2195. $randseed += unpack('S', $str); # unsigned 16 bit value
  2196. }
  2197. srand $randseed;
  2198. if(@testthis && ($testthis[0] ne "")) {
  2199. $TESTCASES=join(" ", @testthis);
  2200. }
  2201. if($valgrind) {
  2202. # we have found valgrind on the host, use it
  2203. # verify that we can invoke it fine
  2204. my $code = runclient("valgrind >/dev/null 2>&1");
  2205. if(($code>>8) != 1) {
  2206. #logmsg "Valgrind failure, disable it\n";
  2207. undef $valgrind;
  2208. } else {
  2209. # since valgrind 2.1.x, '--tool' option is mandatory
  2210. # use it, if it is supported by the version installed on the system
  2211. # (this happened in 2003, so we could probably don't need to care about
  2212. # that old version any longer and just delete this check)
  2213. runclient("valgrind --help 2>&1 | grep -- --tool > /dev/null 2>&1");
  2214. if (($? >> 8)) {
  2215. $valgrind_tool="";
  2216. }
  2217. open(my $curlh, "<", "$CURL");
  2218. my $l = <$curlh>;
  2219. if($l =~ /^\#\!/) {
  2220. # A shell script. This is typically when built with libtool,
  2221. $valgrind="../libtool --mode=execute $valgrind";
  2222. }
  2223. close($curlh);
  2224. # valgrind 3 renamed the --logfile option to --log-file!!!
  2225. # (this happened in 2005, so we could probably don't need to care about
  2226. # that old version any longer and just delete this check)
  2227. my $ver=join(' ', runclientoutput("valgrind --version"));
  2228. # cut off all but digits and dots
  2229. $ver =~ s/[^0-9.]//g;
  2230. if($ver =~ /^(\d+)/) {
  2231. $ver = $1;
  2232. if($ver < 3) {
  2233. $valgrind_logfile="--logfile";
  2234. }
  2235. }
  2236. }
  2237. }
  2238. if ($gdbthis) {
  2239. # open the executable curl and read the first 4 bytes of it
  2240. open(my $check, "<", "$CURL");
  2241. my $c;
  2242. sysread $check, $c, 4;
  2243. close($check);
  2244. if($c eq "#! /") {
  2245. # A shell script. This is typically when built with libtool,
  2246. $libtool = 1;
  2247. $gdb = "../libtool --mode=execute gdb";
  2248. }
  2249. }
  2250. #######################################################################
  2251. # clear and create logging directory:
  2252. #
  2253. # TODO: figure how to get around this. This dir is needed for checksystemfeatures()
  2254. # Maybe create & use & delete a temporary directory in that function
  2255. cleardir($LOGDIR);
  2256. mkdir($LOGDIR, 0777);
  2257. mkdir("$LOGDIR/$LOCKDIR", 0777);
  2258. #######################################################################
  2259. # initialize some variables
  2260. #
  2261. get_disttests();
  2262. if(!$jobs) {
  2263. # Disable buffered logging with only one test job
  2264. setlogfunc(\&logmsg);
  2265. }
  2266. #######################################################################
  2267. # Output curl version and host info being tested
  2268. #
  2269. if(!$listonly) {
  2270. checksystemfeatures();
  2271. }
  2272. #######################################################################
  2273. # initialize configuration needed to set up servers
  2274. # TODO: rearrange things so this can be called only in runner_init()
  2275. #
  2276. initserverconfig();
  2277. if(!$listonly) {
  2278. # these can only be displayed after initserverconfig() has been called
  2279. displayserverfeatures();
  2280. # globally disabled tests
  2281. disabledtests("$TESTDIR/DISABLED");
  2282. }
  2283. #######################################################################
  2284. # Fetch all disabled tests, if there are any
  2285. #
  2286. sub disabledtests {
  2287. my ($file) = @_;
  2288. my @input;
  2289. if(open(my $disabledh, "<", "$file")) {
  2290. while(<$disabledh>) {
  2291. if(/^ *\#/) {
  2292. # allow comments
  2293. next;
  2294. }
  2295. push @input, $_;
  2296. }
  2297. close($disabledh);
  2298. # preprocess the input to make conditionally disabled tests depending
  2299. # on variables
  2300. my @pp = prepro(0, @input);
  2301. for my $t (@pp) {
  2302. if($t =~ /(\d+)/) {
  2303. my ($n) = $1;
  2304. $disabled{$n}=$n; # disable this test number
  2305. if(! -f "$srcdir/data/test$n") {
  2306. print STDERR "WARNING! Non-existing test $n in $file!\n";
  2307. # fail hard to make user notice
  2308. exit 1;
  2309. }
  2310. logmsg "DISABLED: test $n\n" if ($verbose);
  2311. }
  2312. else {
  2313. print STDERR "$file: rubbish content: $t\n";
  2314. exit 2;
  2315. }
  2316. }
  2317. }
  2318. }
  2319. #######################################################################
  2320. # If 'all' tests are requested, find out all test numbers
  2321. #
  2322. if ( $TESTCASES eq "all") {
  2323. # Get all commands and find out their test numbers
  2324. opendir(DIR, $TESTDIR) || die "can't opendir $TESTDIR: $!";
  2325. my @cmds = grep { /^test([0-9]+)$/ && -f "$TESTDIR/$_" } readdir(DIR);
  2326. closedir(DIR);
  2327. $TESTCASES=""; # start with no test cases
  2328. # cut off everything but the digits
  2329. for(@cmds) {
  2330. $_ =~ s/[a-z\/\.]*//g;
  2331. }
  2332. # sort the numbers from low to high
  2333. foreach my $n (sort { $a <=> $b } @cmds) {
  2334. if($disabled{$n}) {
  2335. # skip disabled test cases
  2336. my $why = "configured as DISABLED";
  2337. $skipped{$why}++;
  2338. $teststat[$n]=$why; # store reason for this test case
  2339. next;
  2340. }
  2341. $TESTCASES .= " $n";
  2342. }
  2343. }
  2344. else {
  2345. my $verified="";
  2346. for(split(" ", $TESTCASES)) {
  2347. if (-e "$TESTDIR/test$_") {
  2348. $verified.="$_ ";
  2349. }
  2350. }
  2351. if($verified eq "") {
  2352. print "No existing test cases were specified\n";
  2353. exit;
  2354. }
  2355. $TESTCASES = $verified;
  2356. }
  2357. if($repeat) {
  2358. my $s;
  2359. for(1 .. $repeat) {
  2360. $s .= $TESTCASES;
  2361. }
  2362. $TESTCASES = $s;
  2363. }
  2364. if($scrambleorder) {
  2365. # scramble the order of the test cases
  2366. my @rand;
  2367. while($TESTCASES) {
  2368. my @all = split(/ +/, $TESTCASES);
  2369. if(!$all[0]) {
  2370. # if the first is blank, shift away it
  2371. shift @all;
  2372. }
  2373. my $r = rand @all;
  2374. push @rand, $all[$r];
  2375. $all[$r]="";
  2376. $TESTCASES = join(" ", @all);
  2377. }
  2378. $TESTCASES = join(" ", @rand);
  2379. }
  2380. # Display the contents of the given file. Line endings are canonicalized
  2381. # and excessively long files are elided
  2382. sub displaylogcontent {
  2383. my ($file)=@_;
  2384. if(open(my $single, "<", "$file")) {
  2385. my $linecount = 0;
  2386. my $truncate;
  2387. my @tail;
  2388. while(my $string = <$single>) {
  2389. $string =~ s/\r\n/\n/g;
  2390. $string =~ s/[\r\f\032]/\n/g;
  2391. $string .= "\n" unless ($string =~ /\n$/);
  2392. $string =~ tr/\n//;
  2393. for my $line (split(m/\n/, $string)) {
  2394. $line =~ s/\s*\!$//;
  2395. if ($truncate) {
  2396. push @tail, " $line\n";
  2397. } else {
  2398. logmsg " $line\n";
  2399. }
  2400. $linecount++;
  2401. $truncate = $linecount > 1200;
  2402. }
  2403. }
  2404. close($single);
  2405. if(@tail) {
  2406. my $tailshow = 200;
  2407. my $tailskip = 0;
  2408. my $tailtotal = scalar @tail;
  2409. if($tailtotal > $tailshow) {
  2410. $tailskip = $tailtotal - $tailshow;
  2411. logmsg "=== File too long: $tailskip lines omitted here\n";
  2412. }
  2413. for($tailskip .. $tailtotal-1) {
  2414. logmsg "$tail[$_]";
  2415. }
  2416. }
  2417. }
  2418. }
  2419. sub displaylogs {
  2420. my ($runnerid, $testnum)=@_;
  2421. my $logdir = getrunnerlogdir($runnerid);
  2422. opendir(DIR, "$logdir") ||
  2423. die "can't open dir: $!";
  2424. my @logs = readdir(DIR);
  2425. closedir(DIR);
  2426. logmsg "== Contents of files in the $logdir/ dir after test $testnum\n";
  2427. foreach my $log (sort @logs) {
  2428. if($log =~ /\.(\.|)$/) {
  2429. next; # skip "." and ".."
  2430. }
  2431. if($log =~ /^\.nfs/) {
  2432. next; # skip ".nfs"
  2433. }
  2434. if(($log eq "memdump") || ($log eq "core")) {
  2435. next; # skip "memdump" and "core"
  2436. }
  2437. if((-d "$logdir/$log") || (! -s "$logdir/$log")) {
  2438. next; # skip directory and empty files
  2439. }
  2440. if(($log =~ /^stdout\d+/) && ($log !~ /^stdout$testnum/)) {
  2441. next; # skip stdoutNnn of other tests
  2442. }
  2443. if(($log =~ /^stderr\d+/) && ($log !~ /^stderr$testnum/)) {
  2444. next; # skip stderrNnn of other tests
  2445. }
  2446. if(($log =~ /^upload\d+/) && ($log !~ /^upload$testnum/)) {
  2447. next; # skip uploadNnn of other tests
  2448. }
  2449. if(($log =~ /^curl\d+\.out/) && ($log !~ /^curl$testnum\.out/)) {
  2450. next; # skip curlNnn.out of other tests
  2451. }
  2452. if(($log =~ /^test\d+\.txt/) && ($log !~ /^test$testnum\.txt/)) {
  2453. next; # skip testNnn.txt of other tests
  2454. }
  2455. if(($log =~ /^file\d+\.txt/) && ($log !~ /^file$testnum\.txt/)) {
  2456. next; # skip fileNnn.txt of other tests
  2457. }
  2458. if(($log =~ /^netrc\d+/) && ($log !~ /^netrc$testnum/)) {
  2459. next; # skip netrcNnn of other tests
  2460. }
  2461. if(($log =~ /^trace\d+/) && ($log !~ /^trace$testnum/)) {
  2462. next; # skip traceNnn of other tests
  2463. }
  2464. if(($log =~ /^valgrind\d+/) && ($log !~ /^valgrind$testnum(?:\..*)?$/)) {
  2465. next; # skip valgrindNnn of other tests
  2466. }
  2467. if(($log =~ /^test$testnum$/)) {
  2468. next; # skip test$testnum since it can be very big
  2469. }
  2470. logmsg "=== Start of file $log\n";
  2471. displaylogcontent("$logdir/$log");
  2472. logmsg "=== End of file $log\n";
  2473. }
  2474. }
  2475. #######################################################################
  2476. # Scan tests to find suitable candidates
  2477. #
  2478. my $failed;
  2479. my $failedign;
  2480. my $ok=0;
  2481. my $ign=0;
  2482. my $total=0;
  2483. my $lasttest=0;
  2484. my @at = split(" ", $TESTCASES);
  2485. my $count=0;
  2486. my $endwaitcnt=0;
  2487. $start = time();
  2488. # scan all tests to find ones we should try to run
  2489. my @runtests;
  2490. foreach my $testnum (@at) {
  2491. $lasttest = $testnum if($testnum > $lasttest);
  2492. my ($why, $errorreturncode) = singletest_shouldrun($testnum);
  2493. if($why || $listonly) {
  2494. # Display test name now--test will be completely skipped later
  2495. my $error = singletest_count($testnum, $why);
  2496. next;
  2497. }
  2498. $ignoretestcodes{$testnum} = $errorreturncode;
  2499. push(@runtests, $testnum);
  2500. }
  2501. my $totaltests = scalar(@runtests);
  2502. if($listonly) {
  2503. exit(0);
  2504. }
  2505. #######################################################################
  2506. # Setup CI Test Run
  2507. citest_starttestrun();
  2508. #######################################################################
  2509. # Start test runners
  2510. #
  2511. my $numrunners = $jobs < scalar(@runtests) ? $jobs : scalar(@runtests);
  2512. createrunners($numrunners);
  2513. #######################################################################
  2514. # The main test-loop
  2515. #
  2516. # Every iteration through the loop consists of these steps:
  2517. # - if the global abort flag is set, exit the loop; we are done
  2518. # - if a runner is idle, start a new test on it
  2519. # - if all runners are idle, exit the loop; we are done
  2520. # - if a runner has a response for us, process the response
  2521. # run through each candidate test and execute it
  2522. while () {
  2523. # check the abort flag
  2524. if($globalabort) {
  2525. logmsg singletest_dumplogs();
  2526. logmsg "Aborting tests\n";
  2527. logmsg "Waiting for " . scalar((keys %runnersrunning)) . " outstanding test(s) to finish...\n";
  2528. # Wait for the last requests to complete and throw them away so
  2529. # that IPC calls & responses stay in sync
  2530. # TODO: send a signal to the runners to interrupt a long test
  2531. foreach my $rid (keys %runnersrunning) {
  2532. runnerar($rid);
  2533. delete $runnersrunning{$rid};
  2534. logmsg ".";
  2535. $| = 1;
  2536. }
  2537. logmsg "\n";
  2538. last;
  2539. }
  2540. # Start a new test if possible
  2541. if(scalar(@runnersidle) && scalar(@runtests)) {
  2542. # A runner is ready to run a test, and tests are still available to run
  2543. # so start a new test.
  2544. $count++;
  2545. my $testnum = shift(@runtests);
  2546. # pick a runner for this new test
  2547. my $runnerid = pickrunner($testnum);
  2548. $countforrunner{$runnerid} = $count;
  2549. # Start the test
  2550. my ($error, $again) = singletest($runnerid, $testnum, $countforrunner{$runnerid}, $totaltests);
  2551. if($again) {
  2552. # this runner is busy running a test
  2553. $runnersrunning{$runnerid} = $testnum;
  2554. } else {
  2555. runnerready($runnerid);
  2556. if($error >= 0) {
  2557. # We make this simplifying assumption to avoid having to handle
  2558. # $error properly here, but we must handle the case of runner
  2559. # death without abending here.
  2560. die "Internal error: test must not complete on first call";
  2561. }
  2562. }
  2563. }
  2564. # See if we've completed all the tests
  2565. if(!scalar(%runnersrunning)) {
  2566. # No runners are running; we must be done
  2567. scalar(@runtests) && die 'Internal error: still have tests to run';
  2568. last;
  2569. }
  2570. # See if a test runner needs attention
  2571. # If we could be running more tests, don't wait so we can schedule a new
  2572. # one immediately. If all runners are busy, wait a fraction of a second
  2573. # for one to finish so we can still loop around to check the abort flag.
  2574. my $runnerwait = scalar(@runnersidle) && scalar(@runtests) ? 0 : 0.5;
  2575. my ($ridready, $riderror) = runnerar_ready($runnerwait);
  2576. if($ridready && ! defined $runnersrunning{$ridready}) {
  2577. # On Linux, a closed pipe still shows up as ready instead of error.
  2578. # Detect this here by seeing if we are expecting it to be ready and
  2579. # treat it as an error if not.
  2580. logmsg "ERROR: Runner $ridready is unexpectedly ready; is probably actually dead\n";
  2581. $riderror = $ridready;
  2582. undef $ridready;
  2583. }
  2584. if($ridready) {
  2585. # This runner is ready to be serviced
  2586. my $testnum = $runnersrunning{$ridready};
  2587. defined $testnum || die "Internal error: test for runner $ridready unknown";
  2588. delete $runnersrunning{$ridready};
  2589. my ($error, $again) = singletest($ridready, $testnum, $countforrunner{$ridready}, $totaltests);
  2590. if($again) {
  2591. # this runner is busy running a test
  2592. $runnersrunning{$ridready} = $testnum;
  2593. } else {
  2594. # Test is complete
  2595. runnerready($ridready);
  2596. if($error < 0) {
  2597. # not a test we can run
  2598. next;
  2599. }
  2600. $total++; # number of tests we've run
  2601. if($error>0) {
  2602. if($error==2) {
  2603. # ignored test failures
  2604. $failedign .= "$testnum ";
  2605. }
  2606. else {
  2607. $failed.= "$testnum ";
  2608. }
  2609. if($postmortem) {
  2610. # display all files in $LOGDIR/ in a nice way
  2611. displaylogs($ridready, $testnum);
  2612. }
  2613. if($error==2) {
  2614. $ign++; # ignored test result counter
  2615. }
  2616. elsif(!$anyway) {
  2617. # a test failed, abort
  2618. logmsg "\n - abort tests\n";
  2619. undef @runtests; # empty out the remaining tests
  2620. }
  2621. }
  2622. elsif(!$error) {
  2623. $ok++; # successful test counter
  2624. }
  2625. }
  2626. }
  2627. if($riderror) {
  2628. logmsg "ERROR: runner $riderror is dead! aborting test run\n";
  2629. delete $runnersrunning{$riderror} if(defined $runnersrunning{$riderror});
  2630. $globalabort = 1;
  2631. }
  2632. if(!scalar(@runtests) && ++$endwaitcnt == (240 + $jobs)) {
  2633. # Once all tests have been scheduled on a runner at the end of a test
  2634. # run, we just wait for their results to come in. If we're still
  2635. # waiting after a couple of minutes ($endwaitcnt multiplied by
  2636. # $runnerwait, plus $jobs because that number won't time out), display
  2637. # the same test runner status as we give with a SIGUSR1. This will
  2638. # likely point to a single test that has hung.
  2639. logmsg "Hmmm, the tests are taking a while to finish. Here is the status:\n";
  2640. catch_usr1();
  2641. }
  2642. }
  2643. my $sofar = time() - $start;
  2644. #######################################################################
  2645. # Finish CI Test Run
  2646. citest_finishtestrun();
  2647. # Tests done, stop the servers
  2648. foreach my $runnerid (values %runnerids) {
  2649. runnerac_stopservers($runnerid);
  2650. }
  2651. # Wait for servers to stop
  2652. my $unexpected;
  2653. foreach my $runnerid (values %runnerids) {
  2654. my ($rid, $unexpect, $logs) = runnerar($runnerid);
  2655. $unexpected ||= $unexpect;
  2656. logmsg $logs;
  2657. }
  2658. # Kill the runners
  2659. # There is a race condition here since we don't know exactly when the runners
  2660. # have each finished shutting themselves down, but we're about to exit so it
  2661. # doesn't make much difference.
  2662. foreach my $runnerid (values %runnerids) {
  2663. runnerac_shutdown($runnerid);
  2664. sleep 0; # give runner a context switch so it can shut itself down
  2665. }
  2666. my $numskipped = %skipped ? sum values %skipped : 0;
  2667. my $all = $total + $numskipped;
  2668. runtimestats($lasttest);
  2669. if($all) {
  2670. logmsg "TESTDONE: $all tests were considered during ".
  2671. sprintf("%.0f", $sofar) ." seconds.\n";
  2672. }
  2673. if(%skipped && !$short) {
  2674. my $s=0;
  2675. # Temporary hash to print the restraints sorted by the number
  2676. # of their occurrences
  2677. my %restraints;
  2678. logmsg "TESTINFO: $numskipped tests were skipped due to these restraints:\n";
  2679. for(keys %skipped) {
  2680. my $r = $_;
  2681. my $skip_count = $skipped{$r};
  2682. my $log_line = sprintf("TESTINFO: \"%s\" %d time%s (", $r, $skip_count,
  2683. ($skip_count == 1) ? "" : "s");
  2684. # now gather all test case numbers that had this reason for being
  2685. # skipped
  2686. my $c=0;
  2687. my $max = 9;
  2688. for(0 .. scalar @teststat) {
  2689. my $t = $_;
  2690. if($teststat[$t] && ($teststat[$t] eq $r)) {
  2691. if($c < $max) {
  2692. $log_line .= ", " if($c);
  2693. $log_line .= $t;
  2694. }
  2695. $c++;
  2696. }
  2697. }
  2698. if($c > $max) {
  2699. $log_line .= " and ".($c-$max)." more";
  2700. }
  2701. $log_line .= ")\n";
  2702. $restraints{$log_line} = $skip_count;
  2703. }
  2704. foreach my $log_line (sort {$restraints{$b} <=> $restraints{$a}} keys %restraints) {
  2705. logmsg $log_line;
  2706. }
  2707. }
  2708. if($total) {
  2709. if($failedign) {
  2710. logmsg "IGNORED: failed tests: $failedign\n";
  2711. }
  2712. logmsg sprintf("TESTDONE: $ok tests out of $total reported OK: %d%%\n",
  2713. $ok/$total*100);
  2714. if($failed && ($ok != $total)) {
  2715. logmsg "\nTESTFAIL: These test cases failed: $failed\n\n";
  2716. }
  2717. }
  2718. else {
  2719. logmsg "\nTESTFAIL: No tests were performed\n\n";
  2720. if(scalar(keys %enabled_keywords)) {
  2721. logmsg "TESTFAIL: Nothing matched these keywords: ";
  2722. for(keys %enabled_keywords) {
  2723. logmsg "$_ ";
  2724. }
  2725. logmsg "\n";
  2726. }
  2727. }
  2728. if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
  2729. exit 1;
  2730. }