runtests.pl 100 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065
  1. #!/usr/bin/env perl
  2. #***************************************************************************
  3. # _ _ ____ _
  4. # Project ___| | | | _ \| |
  5. # / __| | | | |_) | |
  6. # | (__| |_| | _ <| |___
  7. # \___|\___/|_| \_\_____|
  8. #
  9. # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  10. #
  11. # This software is licensed as described in the file COPYING, which
  12. # you should have received as part of this distribution. The terms
  13. # are also available at https://curl.se/docs/copyright.html.
  14. #
  15. # You may opt to use, copy, modify, merge, publish, distribute and/or sell
  16. # copies of the Software, and permit persons to whom the Software is
  17. # furnished to do so, under the terms of the COPYING file.
  18. #
  19. # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  20. # KIND, either express or implied.
  21. #
  22. # SPDX-License-Identifier: curl
  23. #
  24. ###########################################################################
  25. # For documentation, run `man ./runtests.1` and see README.md.
  26. # Experimental hooks are available to run tests remotely on machines that
  27. # are able to run curl but are unable to run the test harness.
  28. # The following sections need to be modified:
  29. #
  30. # $HOSTIP, $HOST6IP - Set to the address of the host running the test suite
  31. # $CLIENTIP, $CLIENT6IP - Set to the address of the host running curl
  32. # runclient, runclientoutput - Modify to copy all the files in the log/
  33. # directory to the system running curl, run the given command remotely
  34. # and save the return code or returned stdout (respectively), then
  35. # copy all the files from the remote system's log/ directory back to
  36. # the host running the test suite. This can be done a few ways, such
  37. # as using scp & ssh, rsync & telnet, or using a NFS shared directory
  38. # and ssh.
  39. #
  40. # 'make && make test' needs to be done on both machines before making the
  41. # above changes and running runtests.pl manually. In the shared NFS case,
  42. # the contents of the tests/server/ directory must be from the host
  43. # running the test suite, while the rest must be from the host running curl.
  44. #
  45. # Note that even with these changes a number of tests will still fail (mainly
  46. # to do with cookies, those that set environment variables, or those that
  47. # do more than touch the file system in a <precheck> or <postcheck>
  48. # section). These can be added to the $TESTCASES line below,
  49. # e.g. $TESTCASES="!8 !31 !63 !cookies..."
  50. #
  51. # Finally, to properly support -g and -n, checktestcmd needs to change
  52. # to check the remote system's PATH, and the places in the code where
  53. # the curl binary is read directly to determine its type also need to be
  54. # fixed. As long as the -g option is never given, and the -n is always
  55. # given, this won't be a problem.
  56. use strict;
  57. # Promote all warnings to fatal
  58. use warnings FATAL => 'all';
  59. use 5.006;
  60. use POSIX qw(strftime);
  61. # These should be the only variables that might be needed to get edited:
  62. BEGIN {
  63. # Define srcdir to the location of the tests source directory. This is
  64. # usually set by the Makefile, but for out-of-tree builds with direct
  65. # invocation of runtests.pl, it may not be set.
  66. if(!defined $ENV{'srcdir'}) {
  67. use File::Basename;
  68. $ENV{'srcdir'} = dirname(__FILE__);
  69. }
  70. push(@INC, $ENV{'srcdir'});
  71. # run time statistics needs Time::HiRes
  72. eval {
  73. no warnings "all";
  74. require Time::HiRes;
  75. import Time::HiRes qw( time );
  76. }
  77. }
  78. use Digest::MD5 qw(md5);
  79. use List::Util 'sum';
  80. use pathhelp qw(
  81. exe_ext
  82. sys_native_current_path
  83. );
  84. use processhelp qw(
  85. portable_sleep
  86. );
  87. use appveyor;
  88. use azure;
  89. use getpart; # array functions
  90. use servers;
  91. use valgrind; # valgrind report parser
  92. use globalconfig;
  93. use runner;
  94. use testutil;
  95. my %custom_skip_reasons;
  96. my $ACURL=$VCURL; # what curl binary to use to talk to APIs (relevant for CI)
  97. # ACURL is handy to set to the system one for reliability
  98. my $CURLCONFIG="../curl-config"; # curl-config from current build
  99. # Normally, all test cases should be run, but at times it is handy to
  100. # simply run a particular one:
  101. my $TESTCASES="all";
  102. # To run specific test cases, set them like:
  103. # $TESTCASES="1 2 3 7 8";
  104. #######################################################################
  105. # No variables below this point should need to be modified
  106. #
  107. my $libtool;
  108. my $repeat = 0;
  109. my $start; # time at which testing started
  110. my $uname_release = `uname -r`;
  111. my $is_wsl = $uname_release =~ /Microsoft$/;
  112. my $http_ipv6; # set if HTTP server has IPv6 support
  113. my $http_unix; # set if HTTP server has Unix sockets support
  114. my $ftp_ipv6; # set if FTP server has IPv6 support
  115. my $resolver; # name of the resolver backend (for human presentation)
  116. my $has_textaware; # set if running on a system that has a text mode concept
  117. # on files. Windows for example
  118. my %skipped; # skipped{reason}=counter, reasons for skip
  119. my @teststat; # teststat[testnum]=reason, reasons for skip
  120. my %disabled_keywords; # key words of tests to skip
  121. my %ignored_keywords; # key words of tests to ignore results
  122. my %enabled_keywords; # key words of tests to run
  123. my %disabled; # disabled test cases
  124. my %ignored; # ignored results of test cases
  125. my %ignoretestcodes; # if test results are to be ignored
  126. my $timestats; # time stamping and stats generation
  127. my $fullstats; # show time stats for every single test
  128. my %timeprepini; # timestamp for each test preparation start
  129. my %timesrvrini; # timestamp for each test required servers verification start
  130. my %timesrvrend; # timestamp for each test required servers verification end
  131. my %timetoolini; # timestamp for each test command run starting
  132. my %timetoolend; # timestamp for each test command run stopping
  133. my %timesrvrlog; # timestamp for each test server logs lock removal
  134. my %timevrfyend; # timestamp for each test result verification end
  135. my $globalabort; # flag signalling program abort
  136. # values for $singletest_state
  137. use constant {
  138. ST_INIT => 0,
  139. ST_CLEARLOCKS => 1,
  140. ST_INITED => 2,
  141. ST_PREPROCESS => 3,
  142. ST_RUN => 4,
  143. };
  144. my %singletest_state; # current state of singletest() by runner ID
  145. my %singletest_logs; # log messages while in singletest array ref by runner
  146. my $singletest_bufferedrunner; # runner ID which is buffering logs
  147. my %runnerids; # runner IDs by number
  148. my @runnersidle; # runner IDs idle and ready to execute a test
  149. my %countforrunner; # test count by runner ID
  150. my %runnersrunning; # tests currently running by runner ID
  151. #######################################################################
  152. # variables that command line options may set
  153. #
  154. my $short;
  155. my $no_debuginfod;
  156. my $keepoutfiles; # keep stdout and stderr files after tests
  157. my $clearlocks; # force removal of files by killing locking processes
  158. my $postmortem; # display detailed info about failed tests
  159. my $run_disabled; # run the specific tests even if listed in DISABLED
  160. my $scrambleorder;
  161. my $jobs = 0;
  162. # Azure Pipelines specific variables
  163. my $AZURE_RUN_ID = 0;
  164. my $AZURE_RESULT_ID = 0;
  165. #######################################################################
  166. # logmsg is our general message logging subroutine.
  167. #
  168. sub logmsg {
  169. if($singletest_bufferedrunner) {
  170. # Logs are currently being buffered
  171. return singletest_logmsg(@_);
  172. }
  173. for(@_) {
  174. my $line = $_;
  175. if(!$line) {
  176. next;
  177. }
  178. if ($is_wsl) {
  179. # use \r\n for WSL shell
  180. $line =~ s/\r?\n$/\r\n/g;
  181. }
  182. print "$line";
  183. }
  184. }
  185. #######################################################################
  186. # enable logmsg buffering for the given runner ID
  187. #
  188. sub logmsg_bufferfortest {
  189. my ($runnerid)=@_;
  190. if($jobs) {
  191. # Only enable buffering in multiprocess mode
  192. $singletest_bufferedrunner = $runnerid;
  193. }
  194. }
  195. #######################################################################
  196. # Store a log message in a buffer for this test
  197. # The messages can then be displayed all at once at the end of the test
  198. # which prevents messages from different tests from being interleaved.
  199. sub singletest_logmsg {
  200. if(!exists $singletest_logs{$singletest_bufferedrunner}) {
  201. # initialize to a reference to an empty anonymous array
  202. $singletest_logs{$singletest_bufferedrunner} = [];
  203. }
  204. my $logsref = $singletest_logs{$singletest_bufferedrunner};
  205. push @$logsref, @_;
  206. }
  207. #######################################################################
  208. # Stop buffering log messages, but don't touch them
  209. sub singletest_unbufferlogs {
  210. undef $singletest_bufferedrunner;
  211. }
  212. #######################################################################
  213. # Clear the buffered log messages & stop buffering after returning them
  214. sub singletest_dumplogs {
  215. if(!defined $singletest_bufferedrunner) {
  216. # probably not multiprocess mode and logs weren't buffered
  217. return undef;
  218. }
  219. my $logsref = $singletest_logs{$singletest_bufferedrunner};
  220. my $msg = join("", @$logsref);
  221. delete $singletest_logs{$singletest_bufferedrunner};
  222. singletest_unbufferlogs();
  223. return $msg;
  224. }
  225. sub catch_zap {
  226. my $signame = shift;
  227. print "runtests.pl received SIG$signame, exiting\r\n";
  228. $globalabort = 1;
  229. }
  230. $SIG{INT} = \&catch_zap;
  231. $SIG{TERM} = \&catch_zap;
  232. sub catch_usr1 {
  233. print "runtests.pl internal state:\r\n";
  234. print scalar(%runnersrunning) . " busy test runner(s) of " . scalar(keys %runnerids) . "\r\n";
  235. foreach my $rid (sort(keys(%runnersrunning))) {
  236. my $runnernum = "unknown";
  237. foreach my $rnum (keys %runnerids) {
  238. if($runnerids{$rnum} == $rid) {
  239. $runnernum = $rnum;
  240. last;
  241. }
  242. }
  243. print "Runner $runnernum (id $rid) running test $runnersrunning{$rid} in state $singletest_state{$rid}\r\n";
  244. }
  245. }
  246. eval {
  247. # some msys2 perl versions don't define SIGUSR1
  248. $SIG{USR1} = \&catch_usr1;
  249. };
  250. $SIG{PIPE} = 'IGNORE'; # these errors are captured in the read/write calls
  251. ##########################################################################
  252. # Clear all possible '*_proxy' environment variables for various protocols
  253. # to prevent them to interfere with our testing!
  254. foreach my $protocol (('ftp', 'http', 'ftps', 'https', 'no', 'all')) {
  255. my $proxy = "${protocol}_proxy";
  256. # clear lowercase version
  257. delete $ENV{$proxy} if($ENV{$proxy});
  258. # clear uppercase version
  259. delete $ENV{uc($proxy)} if($ENV{uc($proxy)});
  260. }
  261. # make sure we don't get affected by other variables that control our
  262. # behavior
  263. delete $ENV{'SSL_CERT_DIR'} if($ENV{'SSL_CERT_DIR'});
  264. delete $ENV{'SSL_CERT_PATH'} if($ENV{'SSL_CERT_PATH'});
  265. delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'});
  266. # provide defaults from our config file for ENV vars not explicitly
  267. # set by the caller
  268. if (open(my $fd, "<", "config")) {
  269. while(my $line = <$fd>) {
  270. next if ($line =~ /^#/);
  271. chomp $line;
  272. my ($name, $val) = split(/\s*:\s*/, $line, 2);
  273. $ENV{$name} = $val if(!$ENV{$name});
  274. }
  275. close($fd);
  276. }
  277. # Check if we have nghttpx available and if it talks http/3
  278. my $nghttpx_h3 = 0;
  279. if (!$ENV{"NGHTTPX"}) {
  280. $ENV{"NGHTTPX"} = checktestcmd("nghttpx");
  281. }
  282. if ($ENV{"NGHTTPX"}) {
  283. my $nghttpx_version=join(' ', `"$ENV{'NGHTTPX'}" -v 2>/dev/null`);
  284. $nghttpx_h3 = $nghttpx_version =~ /nghttp3\//;
  285. chomp $nghttpx_h3;
  286. }
  287. #######################################################################
  288. # Get the list of tests that the tests/data/Makefile.am knows about!
  289. #
  290. my $disttests = "";
  291. sub get_disttests {
  292. # If a non-default $TESTDIR is being used there may not be any
  293. # Makefile.inc in which case there's nothing to do.
  294. open(my $dh, "<", "$TESTDIR/Makefile.inc") or return;
  295. while(<$dh>) {
  296. chomp $_;
  297. if(($_ =~ /^#/) ||($_ !~ /test/)) {
  298. next;
  299. }
  300. $disttests .= $_;
  301. }
  302. close($dh);
  303. }
  304. #######################################################################
  305. # Remove all files in the specified directory
  306. #
  307. sub cleardir {
  308. my $dir = $_[0];
  309. my $done = 1; # success
  310. my $file;
  311. # Get all files
  312. opendir(my $dh, $dir) ||
  313. return 0; # can't open dir
  314. while($file = readdir($dh)) {
  315. # Don't clear the $PIDDIR or $LOCKDIR since those need to live beyond
  316. # one test
  317. if(($file !~ /^(\.|\.\.)\z/) &&
  318. "$file" ne $PIDDIR && "$file" ne $LOCKDIR) {
  319. if(-d "$dir/$file") {
  320. if(!cleardir("$dir/$file")) {
  321. $done = 0;
  322. }
  323. if(!rmdir("$dir/$file")) {
  324. $done = 0;
  325. }
  326. }
  327. else {
  328. # Ignore stunnel since we cannot do anything about its locks
  329. if(!unlink("$dir/$file") && "$file" !~ /_stunnel\.log$/) {
  330. $done = 0;
  331. }
  332. }
  333. }
  334. }
  335. closedir $dh;
  336. return $done;
  337. }
  338. #######################################################################
  339. # Given two array references, this function will store them in two temporary
  340. # files, run 'diff' on them, store the result and return the diff output!
  341. sub showdiff {
  342. my ($logdir, $firstref, $secondref)=@_;
  343. my $file1="$logdir/check-generated";
  344. my $file2="$logdir/check-expected";
  345. open(my $temp, ">", "$file1") || die "Failure writing diff file";
  346. for(@$firstref) {
  347. my $l = $_;
  348. $l =~ s/\r/[CR]/g;
  349. $l =~ s/\n/[LF]/g;
  350. $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
  351. print $temp $l;
  352. print $temp "\n";
  353. }
  354. close($temp) || die "Failure writing diff file";
  355. open($temp, ">", "$file2") || die "Failure writing diff file";
  356. for(@$secondref) {
  357. my $l = $_;
  358. $l =~ s/\r/[CR]/g;
  359. $l =~ s/\n/[LF]/g;
  360. $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
  361. print $temp $l;
  362. print $temp "\n";
  363. }
  364. close($temp) || die "Failure writing diff file";
  365. my @out = `diff -u $file2 $file1 2>/dev/null`;
  366. if(!$out[0]) {
  367. @out = `diff -c $file2 $file1 2>/dev/null`;
  368. }
  369. return @out;
  370. }
  371. #######################################################################
  372. # compare test results with the expected output, we might filter off
  373. # some pattern that is allowed to differ, output test results
  374. #
  375. sub compare {
  376. my ($runnerid, $testnum, $testname, $subject, $firstref, $secondref)=@_;
  377. my $result = compareparts($firstref, $secondref);
  378. if($result) {
  379. # timestamp test result verification end
  380. $timevrfyend{$testnum} = Time::HiRes::time();
  381. if(!$short) {
  382. logmsg "\n $testnum: $subject FAILED:\n";
  383. my $logdir = getrunnerlogdir($runnerid);
  384. logmsg showdiff($logdir, $firstref, $secondref);
  385. }
  386. elsif(!$automakestyle) {
  387. logmsg "FAILED\n";
  388. }
  389. else {
  390. # automakestyle
  391. logmsg "FAIL: $testnum - $testname - $subject\n";
  392. }
  393. }
  394. return $result;
  395. }
  396. #######################################################################
  397. # Numeric-sort words in a string
  398. sub numsortwords {
  399. my ($string)=@_;
  400. return join(' ', sort { $a <=> $b } split(' ', $string));
  401. }
  402. #######################################################################
  403. # Parse and store the protocols in curl's Protocols: line
  404. sub parseprotocols {
  405. my ($line)=@_;
  406. @protocols = split(' ', lc($line));
  407. # Generate a "proto-ipv6" version of each protocol to match the
  408. # IPv6 <server> name and a "proto-unix" to match the variant which
  409. # uses Unix domain sockets. This works even if support isn't
  410. # compiled in because the <features> test will fail.
  411. push @protocols, map(("$_-ipv6", "$_-unix"), @protocols);
  412. # 'http-proxy' is used in test cases to do CONNECT through
  413. push @protocols, 'http-proxy';
  414. # 'none' is used in test cases to mean no server
  415. push @protocols, 'none';
  416. }
  417. #######################################################################
  418. # Check & display information about curl and the host the test suite runs on.
  419. # Information to do with servers is displayed in displayserverfeatures, after
  420. # the server initialization is performed.
  421. sub checksystemfeatures {
  422. my $feat;
  423. my $curl;
  424. my $libcurl;
  425. my $versretval;
  426. my $versnoexec;
  427. my @version=();
  428. my @disabled;
  429. my $dis = "";
  430. my $curlverout="$LOGDIR/curlverout.log";
  431. my $curlvererr="$LOGDIR/curlvererr.log";
  432. my $versioncmd=shell_quote($CURL) . " --version 1>$curlverout 2>$curlvererr";
  433. unlink($curlverout);
  434. unlink($curlvererr);
  435. $versretval = runclient($versioncmd);
  436. $versnoexec = $!;
  437. $DATE = strftime "%Y-%m-%d", localtime;
  438. open(my $versout, "<", "$curlverout");
  439. @version = <$versout>;
  440. close($versout);
  441. open(my $disabledh, "-|", "server/disabled".exe_ext('TOOL'));
  442. @disabled = <$disabledh>;
  443. close($disabledh);
  444. if($disabled[0]) {
  445. s/[\r\n]//g for @disabled;
  446. $dis = join(", ", @disabled);
  447. }
  448. $resolver="stock";
  449. for(@version) {
  450. chomp;
  451. if($_ =~ /^curl ([^ ]*)/) {
  452. $curl = $_;
  453. $CURLVERSION = $1;
  454. $CURLVERNUM = $CURLVERSION;
  455. $CURLVERNUM =~ s/^([0-9.]+)(.*)/$1/; # leading dots and numbers
  456. $curl =~ s/^(.*)(libcurl.*)/$1/g || die "Failure determining curl binary version";
  457. $libcurl = $2;
  458. if($curl =~ /linux|bsd|solaris/) {
  459. # system support LD_PRELOAD; may be disabled later
  460. $feature{"ld_preload"} = 1;
  461. }
  462. if($curl =~ /win32|Windows|mingw(32|64)/) {
  463. # This is a Windows MinGW build or native build, we need to use
  464. # Win32-style path.
  465. $pwd = sys_native_current_path();
  466. $has_textaware = 1;
  467. $feature{"win32"} = 1;
  468. # set if built with MinGW (as opposed to MinGW-w64)
  469. $feature{"MinGW"} = 1 if ($curl =~ /-pc-mingw32/);
  470. }
  471. if ($libcurl =~ /\s(winssl|schannel)\b/i) {
  472. $feature{"Schannel"} = 1;
  473. $feature{"SSLpinning"} = 1;
  474. }
  475. elsif ($libcurl =~ /\sopenssl\b/i) {
  476. $feature{"OpenSSL"} = 1;
  477. $feature{"SSLpinning"} = 1;
  478. }
  479. elsif ($libcurl =~ /\sgnutls\b/i) {
  480. $feature{"GnuTLS"} = 1;
  481. $feature{"SSLpinning"} = 1;
  482. }
  483. elsif ($libcurl =~ /\srustls-ffi\b/i) {
  484. $feature{"rustls"} = 1;
  485. }
  486. elsif ($libcurl =~ /\swolfssl\b/i) {
  487. $feature{"wolfssl"} = 1;
  488. $feature{"SSLpinning"} = 1;
  489. }
  490. elsif ($libcurl =~ /\sbearssl\b/i) {
  491. $feature{"bearssl"} = 1;
  492. }
  493. elsif ($libcurl =~ /\ssecuretransport\b/i) {
  494. $feature{"sectransp"} = 1;
  495. $feature{"SSLpinning"} = 1;
  496. }
  497. elsif ($libcurl =~ /\sBoringSSL\b/i) {
  498. # OpenSSL compatible API
  499. $feature{"OpenSSL"} = 1;
  500. $feature{"SSLpinning"} = 1;
  501. }
  502. elsif ($libcurl =~ /\slibressl\b/i) {
  503. # OpenSSL compatible API
  504. $feature{"OpenSSL"} = 1;
  505. $feature{"SSLpinning"} = 1;
  506. }
  507. elsif ($libcurl =~ /\smbedTLS\b/i) {
  508. $feature{"mbedtls"} = 1;
  509. $feature{"SSLpinning"} = 1;
  510. }
  511. if ($libcurl =~ /ares/i) {
  512. $feature{"c-ares"} = 1;
  513. $resolver="c-ares";
  514. }
  515. if ($libcurl =~ /Hyper/i) {
  516. $feature{"hyper"} = 1;
  517. }
  518. if ($libcurl =~ /nghttp2/i) {
  519. # nghttp2 supports h2c, hyper does not
  520. $feature{"h2c"} = 1;
  521. }
  522. if ($libcurl =~ /libssh2/i) {
  523. $feature{"libssh2"} = 1;
  524. }
  525. if ($libcurl =~ /libssh\/([0-9.]*)\//i) {
  526. $feature{"libssh"} = 1;
  527. if($1 =~ /(\d+)\.(\d+).(\d+)/) {
  528. my $v = $1 * 100 + $2 * 10 + $3;
  529. if($v < 94) {
  530. # before 0.9.4
  531. $feature{"oldlibssh"} = 1;
  532. }
  533. }
  534. }
  535. if ($libcurl =~ /wolfssh/i) {
  536. $feature{"wolfssh"} = 1;
  537. }
  538. }
  539. elsif($_ =~ /^Protocols: (.*)/i) {
  540. # these are the protocols compiled in to this libcurl
  541. parseprotocols($1);
  542. }
  543. elsif($_ =~ /^Features: (.*)/i) {
  544. $feat = $1;
  545. # built with memory tracking support (--enable-curldebug); may be disabled later
  546. $feature{"TrackMemory"} = $feat =~ /TrackMemory/i;
  547. # curl was built with --enable-debug
  548. $feature{"debug"} = $feat =~ /debug/i;
  549. # ssl enabled
  550. $feature{"SSL"} = $feat =~ /SSL/i;
  551. # multiple ssl backends available.
  552. $feature{"MultiSSL"} = $feat =~ /MultiSSL/i;
  553. # large file support
  554. $feature{"large_file"} = $feat =~ /Largefile/i;
  555. # IDN support
  556. $feature{"idn"} = $feat =~ /IDN/i;
  557. # IPv6 support
  558. $feature{"ipv6"} = $feat =~ /IPv6/i;
  559. # Unix sockets support
  560. $feature{"unix-sockets"} = $feat =~ /UnixSockets/i;
  561. # libz compression
  562. $feature{"libz"} = $feat =~ /libz/i;
  563. # Brotli compression
  564. $feature{"brotli"} = $feat =~ /brotli/i;
  565. # Zstd compression
  566. $feature{"zstd"} = $feat =~ /zstd/i;
  567. # NTLM enabled
  568. $feature{"NTLM"} = $feat =~ /NTLM/i;
  569. # NTLM delegation to winbind daemon ntlm_auth helper enabled
  570. $feature{"NTLM_WB"} = $feat =~ /NTLM_WB/i;
  571. # SSPI enabled
  572. $feature{"SSPI"} = $feat =~ /SSPI/i;
  573. # GSS-API enabled
  574. $feature{"GSS-API"} = $feat =~ /GSS-API/i;
  575. # Kerberos enabled
  576. $feature{"Kerberos"} = $feat =~ /Kerberos/i;
  577. # SPNEGO enabled
  578. $feature{"SPNEGO"} = $feat =~ /SPNEGO/i;
  579. # CharConv enabled
  580. $feature{"CharConv"} = $feat =~ /CharConv/i;
  581. # TLS-SRP enabled
  582. $feature{"TLS-SRP"} = $feat =~ /TLS-SRP/i;
  583. # PSL enabled
  584. $feature{"PSL"} = $feat =~ /PSL/i;
  585. # alt-svc enabled
  586. $feature{"alt-svc"} = $feat =~ /alt-svc/i;
  587. # HSTS support
  588. $feature{"HSTS"} = $feat =~ /HSTS/i;
  589. if($feat =~ /AsynchDNS/i) {
  590. if(!$feature{"c-ares"}) {
  591. # this means threaded resolver
  592. $feature{"threaded-resolver"} = 1;
  593. $resolver="threaded";
  594. }
  595. }
  596. # http2 enabled
  597. $feature{"http/2"} = $feat =~ /HTTP2/;
  598. if($feature{"http/2"}) {
  599. push @protocols, 'http/2';
  600. }
  601. # http3 enabled
  602. $feature{"http/3"} = $feat =~ /HTTP3/;
  603. if($feature{"http/3"}) {
  604. push @protocols, 'http/3';
  605. }
  606. # https proxy support
  607. $feature{"https-proxy"} = $feat =~ /HTTPS-proxy/;
  608. if($feature{"https-proxy"}) {
  609. # 'https-proxy' is used as "server" so consider it a protocol
  610. push @protocols, 'https-proxy';
  611. }
  612. # UNICODE support
  613. $feature{"Unicode"} = $feat =~ /Unicode/i;
  614. # Thread-safe init
  615. $feature{"threadsafe"} = $feat =~ /threadsafe/i;
  616. }
  617. #
  618. # Test harness currently uses a non-stunnel server in order to
  619. # run HTTP TLS-SRP tests required when curl is built with https
  620. # protocol support and TLS-SRP feature enabled. For convenience
  621. # 'httptls' may be included in the test harness protocols array
  622. # to differentiate this from classic stunnel based 'https' test
  623. # harness server.
  624. #
  625. if($feature{"TLS-SRP"}) {
  626. my $add_httptls;
  627. for(@protocols) {
  628. if($_ =~ /^https(-ipv6|)$/) {
  629. $add_httptls=1;
  630. last;
  631. }
  632. }
  633. if($add_httptls && (! grep /^httptls$/, @protocols)) {
  634. push @protocols, 'httptls';
  635. push @protocols, 'httptls-ipv6';
  636. }
  637. }
  638. }
  639. if(!$curl) {
  640. logmsg "unable to get curl's version, further details are:\n";
  641. logmsg "issued command: \n";
  642. logmsg "$versioncmd \n";
  643. if ($versretval == -1) {
  644. logmsg "command failed with: \n";
  645. logmsg "$versnoexec \n";
  646. }
  647. elsif ($versretval & 127) {
  648. logmsg sprintf("command died with signal %d, and %s coredump.\n",
  649. ($versretval & 127), ($versretval & 128)?"a":"no");
  650. }
  651. else {
  652. logmsg sprintf("command exited with value %d \n", $versretval >> 8);
  653. }
  654. logmsg "contents of $curlverout: \n";
  655. displaylogcontent("$curlverout");
  656. logmsg "contents of $curlvererr: \n";
  657. displaylogcontent("$curlvererr");
  658. die "couldn't get curl's version";
  659. }
  660. if(-r "../lib/curl_config.h") {
  661. open(my $conf, "<", "../lib/curl_config.h");
  662. while(<$conf>) {
  663. if($_ =~ /^\#define HAVE_GETRLIMIT/) {
  664. # set if system has getrlimit()
  665. $feature{"getrlimit"} = 1;
  666. }
  667. }
  668. close($conf);
  669. }
  670. # allow this feature only if debug mode is disabled
  671. $feature{"ld_preload"} = $feature{"ld_preload"} && !$feature{"debug"};
  672. if($feature{"ipv6"}) {
  673. # client has IPv6 support
  674. # check if the HTTP server has it!
  675. my $cmd = "server/sws".exe_ext('SRV')." --version";
  676. my @sws = `$cmd`;
  677. if($sws[0] =~ /IPv6/) {
  678. # HTTP server has IPv6 support!
  679. $http_ipv6 = 1;
  680. }
  681. # check if the FTP server has it!
  682. $cmd = "server/sockfilt".exe_ext('SRV')." --version";
  683. @sws = `$cmd`;
  684. if($sws[0] =~ /IPv6/) {
  685. # FTP server has IPv6 support!
  686. $ftp_ipv6 = 1;
  687. }
  688. }
  689. if($feature{"unix-sockets"}) {
  690. # client has Unix sockets support, check whether the HTTP server has it
  691. my $cmd = "server/sws".exe_ext('SRV')." --version";
  692. my @sws = `$cmd`;
  693. $http_unix = 1 if($sws[0] =~ /unix/);
  694. }
  695. open(my $manh, "-|", shell_quote($CURL) . " -M 2>&1");
  696. while(my $s = <$manh>) {
  697. if($s =~ /built-in manual was disabled at build-time/) {
  698. $feature{"manual"} = 0;
  699. last;
  700. }
  701. $feature{"manual"} = 1;
  702. last;
  703. }
  704. close($manh);
  705. $feature{"unittest"} = $feature{"debug"};
  706. $feature{"nghttpx"} = !!$ENV{'NGHTTPX'};
  707. $feature{"nghttpx-h3"} = !!$nghttpx_h3;
  708. #
  709. # strings that must exactly match the names used in server/disabled.c
  710. #
  711. $feature{"cookies"} = 1;
  712. # Use this as a proxy for any cryptographic authentication
  713. $feature{"crypto"} = $feature{"NTLM"} || $feature{"Kerberos"} || $feature{"SPNEGO"};
  714. $feature{"DoH"} = 1;
  715. $feature{"HTTP-auth"} = 1;
  716. $feature{"Mime"} = 1;
  717. $feature{"form-api"} = 1;
  718. $feature{"netrc"} = 1;
  719. $feature{"parsedate"} = 1;
  720. $feature{"proxy"} = 1;
  721. $feature{"shuffle-dns"} = 1;
  722. $feature{"typecheck"} = 1;
  723. $feature{"verbose-strings"} = 1;
  724. $feature{"wakeup"} = 1;
  725. $feature{"headers-api"} = 1;
  726. $feature{"xattr"} = 1;
  727. $feature{"large-time"} = 1;
  728. $feature{"sha512-256"} = 1;
  729. # make each protocol an enabled "feature"
  730. for my $p (@protocols) {
  731. $feature{$p} = 1;
  732. }
  733. # 'socks' was once here but is now removed
  734. $has_shared = `sh $CURLCONFIG --built-shared`;
  735. chomp $has_shared;
  736. $has_shared = $has_shared eq "yes";
  737. if(!$feature{"TrackMemory"} && $torture) {
  738. die "can't run torture tests since curl was built without ".
  739. "TrackMemory feature (--enable-curldebug)";
  740. }
  741. my $hostname=join(' ', runclientoutput("hostname"));
  742. my $hosttype=join(' ', runclientoutput("uname -a"));
  743. my $hostos=$^O;
  744. # display summary information about curl and the test host
  745. logmsg ("********* System characteristics ******** \n",
  746. "* $curl\n",
  747. "* $libcurl\n",
  748. "* Features: $feat\n",
  749. "* Disabled: $dis\n",
  750. "* Host: $hostname",
  751. "* System: $hosttype",
  752. "* OS: $hostos\n");
  753. if($jobs) {
  754. # Only show if not the default for now
  755. logmsg "* Jobs: $jobs\n";
  756. }
  757. if($feature{"TrackMemory"} && $feature{"threaded-resolver"}) {
  758. logmsg("*\n",
  759. "*** DISABLES memory tracking when using threaded resolver\n",
  760. "*\n");
  761. }
  762. logmsg sprintf("* Env: %s%s%s", $valgrind?"Valgrind ":"",
  763. $run_event_based?"event-based ":"",
  764. $nghttpx_h3);
  765. logmsg sprintf("%s\n", $libtool?"Libtool ":"");
  766. logmsg ("* Seed: $randseed\n");
  767. # Disable memory tracking when using threaded resolver
  768. $feature{"TrackMemory"} = $feature{"TrackMemory"} && !$feature{"threaded-resolver"};
  769. # toggle off the features that were disabled in the build
  770. for my $d(@disabled) {
  771. $feature{$d} = 0;
  772. }
  773. }
  774. #######################################################################
  775. # display information about server features
  776. #
  777. sub displayserverfeatures {
  778. logmsg sprintf("* Servers: %s", $stunnel?"SSL ":"");
  779. logmsg sprintf("%s", $http_ipv6?"HTTP-IPv6 ":"");
  780. logmsg sprintf("%s", $http_unix?"HTTP-unix ":"");
  781. logmsg sprintf("%s\n", $ftp_ipv6?"FTP-IPv6 ":"");
  782. logmsg "***************************************** \n";
  783. }
  784. #######################################################################
  785. # Provide time stamps for single test skipped events
  786. #
  787. sub timestampskippedevents {
  788. my $testnum = $_[0];
  789. return if((not defined($testnum)) || ($testnum < 1));
  790. if($timestats) {
  791. if($timevrfyend{$testnum}) {
  792. return;
  793. }
  794. elsif($timesrvrlog{$testnum}) {
  795. $timevrfyend{$testnum} = $timesrvrlog{$testnum};
  796. return;
  797. }
  798. elsif($timetoolend{$testnum}) {
  799. $timevrfyend{$testnum} = $timetoolend{$testnum};
  800. $timesrvrlog{$testnum} = $timetoolend{$testnum};
  801. }
  802. elsif($timetoolini{$testnum}) {
  803. $timevrfyend{$testnum} = $timetoolini{$testnum};
  804. $timesrvrlog{$testnum} = $timetoolini{$testnum};
  805. $timetoolend{$testnum} = $timetoolini{$testnum};
  806. }
  807. elsif($timesrvrend{$testnum}) {
  808. $timevrfyend{$testnum} = $timesrvrend{$testnum};
  809. $timesrvrlog{$testnum} = $timesrvrend{$testnum};
  810. $timetoolend{$testnum} = $timesrvrend{$testnum};
  811. $timetoolini{$testnum} = $timesrvrend{$testnum};
  812. }
  813. elsif($timesrvrini{$testnum}) {
  814. $timevrfyend{$testnum} = $timesrvrini{$testnum};
  815. $timesrvrlog{$testnum} = $timesrvrini{$testnum};
  816. $timetoolend{$testnum} = $timesrvrini{$testnum};
  817. $timetoolini{$testnum} = $timesrvrini{$testnum};
  818. $timesrvrend{$testnum} = $timesrvrini{$testnum};
  819. }
  820. elsif($timeprepini{$testnum}) {
  821. $timevrfyend{$testnum} = $timeprepini{$testnum};
  822. $timesrvrlog{$testnum} = $timeprepini{$testnum};
  823. $timetoolend{$testnum} = $timeprepini{$testnum};
  824. $timetoolini{$testnum} = $timeprepini{$testnum};
  825. $timesrvrend{$testnum} = $timeprepini{$testnum};
  826. $timesrvrini{$testnum} = $timeprepini{$testnum};
  827. }
  828. }
  829. }
  830. # Setup CI Test Run
  831. sub citest_starttestrun {
  832. if(azure_check_environment()) {
  833. $AZURE_RUN_ID = azure_create_test_run($ACURL);
  834. logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose);
  835. }
  836. # Appveyor doesn't require anything here
  837. }
  838. # Register the test case with the CI runner
  839. sub citest_starttest {
  840. my $testnum = $_[0];
  841. # get the name of the test early
  842. my $testname= (getpart("client", "name"))[0];
  843. chomp $testname;
  844. # create test result in CI services
  845. if(azure_check_environment() && $AZURE_RUN_ID) {
  846. $AZURE_RESULT_ID = azure_create_test_result($ACURL, $AZURE_RUN_ID, $testnum, $testname);
  847. }
  848. elsif(appveyor_check_environment()) {
  849. appveyor_create_test_result($ACURL, $testnum, $testname);
  850. }
  851. }
  852. # Submit the test case result with the CI runner
  853. sub citest_finishtest {
  854. my ($testnum, $error) = @_;
  855. # update test result in CI services
  856. if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
  857. $AZURE_RESULT_ID = azure_update_test_result($ACURL, $AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
  858. $timeprepini{$testnum}, $timevrfyend{$testnum});
  859. }
  860. elsif(appveyor_check_environment()) {
  861. appveyor_update_test_result($ACURL, $testnum, $error, $timeprepini{$testnum}, $timevrfyend{$testnum});
  862. }
  863. }
  864. # Complete CI test run
  865. sub citest_finishtestrun {
  866. if(azure_check_environment() && $AZURE_RUN_ID) {
  867. $AZURE_RUN_ID = azure_update_test_run($ACURL, $AZURE_RUN_ID);
  868. }
  869. # Appveyor doesn't require anything here
  870. }
  871. # add one set of test timings from the runner to global set
  872. sub updatetesttimings {
  873. my ($testnum, %testtimings)=@_;
  874. if(defined $testtimings{"timeprepini"}) {
  875. $timeprepini{$testnum} = $testtimings{"timeprepini"};
  876. }
  877. if(defined $testtimings{"timesrvrini"}) {
  878. $timesrvrini{$testnum} = $testtimings{"timesrvrini"};
  879. }
  880. if(defined $testtimings{"timesrvrend"}) {
  881. $timesrvrend{$testnum} = $testtimings{"timesrvrend"};
  882. }
  883. if(defined $testtimings{"timetoolini"}) {
  884. $timetoolini{$testnum} = $testtimings{"timetoolini"};
  885. }
  886. if(defined $testtimings{"timetoolend"}) {
  887. $timetoolend{$testnum} = $testtimings{"timetoolend"};
  888. }
  889. if(defined $testtimings{"timesrvrlog"}) {
  890. $timesrvrlog{$testnum} = $testtimings{"timesrvrlog"};
  891. }
  892. }
  893. #######################################################################
  894. # Return the log directory for the given test runner
  895. sub getrunnernumlogdir {
  896. my $runnernum = $_[0];
  897. return $jobs > 1 ? "$LOGDIR/$runnernum" : $LOGDIR;
  898. }
  899. #######################################################################
  900. # Return the log directory for the given test runner ID
  901. sub getrunnerlogdir {
  902. my $runnerid = $_[0];
  903. if($jobs <= 1) {
  904. return $LOGDIR;
  905. }
  906. # TODO: speed up this O(n) operation
  907. for my $runnernum (keys %runnerids) {
  908. if($runnerid eq $runnerids{$runnernum}) {
  909. return "$LOGDIR/$runnernum";
  910. }
  911. }
  912. die "Internal error: runner ID $runnerid not found";
  913. }
  914. #######################################################################
  915. # Verify that this test case should be run
  916. sub singletest_shouldrun {
  917. my $testnum = $_[0];
  918. my $why; # why the test won't be run
  919. my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
  920. my @what; # what features are needed
  921. if($disttests !~ /test$testnum(\W|\z)/ ) {
  922. logmsg "Warning: test$testnum not present in tests/data/Makefile.inc\n";
  923. }
  924. if($disabled{$testnum}) {
  925. if(!$run_disabled) {
  926. $why = "listed in DISABLED";
  927. }
  928. else {
  929. logmsg "Warning: test$testnum is explicitly disabled\n";
  930. }
  931. }
  932. if($ignored{$testnum}) {
  933. logmsg "Warning: test$testnum result is ignored\n";
  934. $errorreturncode = 2;
  935. }
  936. if(loadtest("${TESTDIR}/test${testnum}")) {
  937. if($verbose) {
  938. # this is not a test
  939. logmsg "RUN: $testnum doesn't look like a test case\n";
  940. }
  941. $why = "no test";
  942. }
  943. else {
  944. @what = getpart("client", "features");
  945. }
  946. # We require a feature to be present
  947. for(@what) {
  948. my $f = $_;
  949. $f =~ s/\s//g;
  950. if($f =~ /^([^!].*)$/) {
  951. if($feature{$1}) {
  952. next;
  953. }
  954. $why = "curl lacks $1 support";
  955. last;
  956. }
  957. }
  958. # We require a feature to not be present
  959. if(!$why) {
  960. for(@what) {
  961. my $f = $_;
  962. $f =~ s/\s//g;
  963. if($f =~ /^!(.*)$/) {
  964. if(!$feature{$1}) {
  965. next;
  966. }
  967. }
  968. else {
  969. next;
  970. }
  971. $why = "curl has $1 support";
  972. last;
  973. }
  974. }
  975. my @info_keywords;
  976. if(!$why) {
  977. @info_keywords = getpart("info", "keywords");
  978. if(!$info_keywords[0]) {
  979. $why = "missing the <keywords> section!";
  980. }
  981. my $match;
  982. for my $k (@info_keywords) {
  983. chomp $k;
  984. if ($disabled_keywords{lc($k)}) {
  985. $why = "disabled by keyword";
  986. }
  987. elsif ($enabled_keywords{lc($k)}) {
  988. $match = 1;
  989. }
  990. if ($ignored_keywords{lc($k)}) {
  991. logmsg "Warning: test$testnum result is ignored due to $k\n";
  992. $errorreturncode = 2;
  993. }
  994. }
  995. if(!$why && !$match && %enabled_keywords) {
  996. $why = "disabled by missing keyword";
  997. }
  998. }
  999. if (!$why && defined $custom_skip_reasons{test}{$testnum}) {
  1000. $why = $custom_skip_reasons{test}{$testnum};
  1001. }
  1002. if (!$why && defined $custom_skip_reasons{tool}) {
  1003. foreach my $tool (getpart("client", "tool")) {
  1004. foreach my $tool_skip_pattern (keys %{$custom_skip_reasons{tool}}) {
  1005. if ($tool =~ /$tool_skip_pattern/i) {
  1006. $why = $custom_skip_reasons{tool}{$tool_skip_pattern};
  1007. }
  1008. }
  1009. }
  1010. }
  1011. if (!$why && defined $custom_skip_reasons{keyword}) {
  1012. foreach my $keyword (@info_keywords) {
  1013. foreach my $keyword_skip_pattern (keys %{$custom_skip_reasons{keyword}}) {
  1014. if ($keyword =~ /$keyword_skip_pattern/i) {
  1015. $why = $custom_skip_reasons{keyword}{$keyword_skip_pattern};
  1016. }
  1017. }
  1018. }
  1019. }
  1020. return ($why, $errorreturncode);
  1021. }
  1022. #######################################################################
  1023. # Print the test name and count tests
  1024. sub singletest_count {
  1025. my ($testnum, $why) = @_;
  1026. if($why && !$listonly) {
  1027. # there's a problem, count it as "skipped"
  1028. $skipped{$why}++;
  1029. $teststat[$testnum]=$why; # store reason for this test case
  1030. if(!$short) {
  1031. if($skipped{$why} <= 3) {
  1032. # show only the first three skips for each reason
  1033. logmsg sprintf("test %04d SKIPPED: $why\n", $testnum);
  1034. }
  1035. }
  1036. timestampskippedevents($testnum);
  1037. return -1;
  1038. }
  1039. # At this point we've committed to run this test
  1040. logmsg sprintf("test %04d...", $testnum) if(!$automakestyle);
  1041. # name of the test
  1042. my $testname= (getpart("client", "name"))[0];
  1043. chomp $testname;
  1044. logmsg "[$testname]\n" if(!$short);
  1045. if($listonly) {
  1046. timestampskippedevents($testnum);
  1047. }
  1048. return 0;
  1049. }
  1050. #######################################################################
  1051. # Verify test succeeded
  1052. sub singletest_check {
  1053. my ($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind)=@_;
  1054. # Skip all the verification on torture tests
  1055. if ($torture) {
  1056. # timestamp test result verification end
  1057. $timevrfyend{$testnum} = Time::HiRes::time();
  1058. return -2;
  1059. }
  1060. my $logdir = getrunnerlogdir($runnerid);
  1061. my @err = getpart("verify", "errorcode");
  1062. my $errorcode = $err[0] || "0";
  1063. my $ok="";
  1064. my $res;
  1065. chomp $errorcode;
  1066. my $testname= (getpart("client", "name"))[0];
  1067. chomp $testname;
  1068. # what parts to cut off from stdout/stderr
  1069. my @stripfile = getpart("verify", "stripfile");
  1070. my @validstdout = getpart("verify", "stdout");
  1071. # get all attributes
  1072. my %hash = getpartattr("verify", "stdout");
  1073. my $loadfile = $hash{'loadfile'};
  1074. if ($loadfile) {
  1075. open(my $tmp, "<", "$loadfile") || die "Cannot open file $loadfile: $!";
  1076. @validstdout = <$tmp>;
  1077. close($tmp);
  1078. # Enforce LF newlines on load
  1079. s/\r\n/\n/g for @validstdout;
  1080. }
  1081. if (@validstdout) {
  1082. # verify redirected stdout
  1083. my @actual = loadarray(stdoutfilename($logdir, $testnum));
  1084. foreach my $strip (@stripfile) {
  1085. chomp $strip;
  1086. my @newgen;
  1087. for(@actual) {
  1088. eval $strip;
  1089. if($_) {
  1090. push @newgen, $_;
  1091. }
  1092. }
  1093. # this is to get rid of array entries that vanished (zero
  1094. # length) because of replacements
  1095. @actual = @newgen;
  1096. }
  1097. # get the mode attribute
  1098. my $filemode=$hash{'mode'};
  1099. if($filemode && ($filemode eq "text") && $has_textaware) {
  1100. # text mode when running on windows: fix line endings
  1101. s/\r\n/\n/g for @validstdout;
  1102. s/\n/\r\n/g for @validstdout;
  1103. s/\r\n/\n/g for @actual;
  1104. s/\n/\r\n/g for @actual;
  1105. }
  1106. if($hash{'nonewline'}) {
  1107. # Yes, we must cut off the final newline from the final line
  1108. # of the protocol data
  1109. chomp($validstdout[-1]);
  1110. }
  1111. if($hash{'crlf'} ||
  1112. ($feature{"hyper"} && ($keywords{"HTTP"}
  1113. || $keywords{"HTTPS"}))) {
  1114. subnewlines(0, \$_) for @validstdout;
  1115. }
  1116. $res = compare($runnerid, $testnum, $testname, "stdout", \@actual, \@validstdout);
  1117. if($res) {
  1118. return -1;
  1119. }
  1120. $ok .= "s";
  1121. }
  1122. else {
  1123. $ok .= "-"; # stdout not checked
  1124. }
  1125. my @validstderr = getpart("verify", "stderr");
  1126. if (@validstderr) {
  1127. # verify redirected stderr
  1128. my @actual = loadarray(stderrfilename($logdir, $testnum));
  1129. foreach my $strip (@stripfile) {
  1130. chomp $strip;
  1131. my @newgen;
  1132. for(@actual) {
  1133. eval $strip;
  1134. if($_) {
  1135. push @newgen, $_;
  1136. }
  1137. }
  1138. # this is to get rid of array entries that vanished (zero
  1139. # length) because of replacements
  1140. @actual = @newgen;
  1141. }
  1142. # get all attributes
  1143. my %hash = getpartattr("verify", "stderr");
  1144. # get the mode attribute
  1145. my $filemode=$hash{'mode'};
  1146. if($filemode && ($filemode eq "text") && $feature{"hyper"}) {
  1147. # text mode check in hyper-mode. Sometimes necessary if the stderr
  1148. # data *looks* like HTTP and thus has gotten CRLF newlines
  1149. # mistakenly
  1150. s/\r\n/\n/g for @validstderr;
  1151. }
  1152. if($filemode && ($filemode eq "text") && $has_textaware) {
  1153. # text mode when running on windows: fix line endings
  1154. s/\r\n/\n/g for @validstderr;
  1155. s/\n/\r\n/g for @validstderr;
  1156. }
  1157. if($hash{'nonewline'}) {
  1158. # Yes, we must cut off the final newline from the final line
  1159. # of the protocol data
  1160. chomp($validstderr[-1]);
  1161. }
  1162. if($hash{'crlf'}) {
  1163. subnewlines(0, \$_) for @validstderr;
  1164. }
  1165. $res = compare($runnerid, $testnum, $testname, "stderr", \@actual, \@validstderr);
  1166. if($res) {
  1167. return -1;
  1168. }
  1169. $ok .= "r";
  1170. }
  1171. else {
  1172. $ok .= "-"; # stderr not checked
  1173. }
  1174. # what to cut off from the live protocol sent by curl
  1175. my @strip = getpart("verify", "strip");
  1176. # what parts to cut off from the protocol & upload
  1177. my @strippart = getpart("verify", "strippart");
  1178. # this is the valid protocol blurb curl should generate
  1179. my @protocol= getpart("verify", "protocol");
  1180. if(@protocol) {
  1181. # Verify the sent request
  1182. my @out = loadarray("$logdir/$SERVERIN");
  1183. # check if there's any attributes on the verify/protocol section
  1184. my %hash = getpartattr("verify", "protocol");
  1185. if($hash{'nonewline'}) {
  1186. # Yes, we must cut off the final newline from the final line
  1187. # of the protocol data
  1188. chomp($protocol[-1]);
  1189. }
  1190. for(@strip) {
  1191. # strip off all lines that match the patterns from both arrays
  1192. chomp $_;
  1193. @out = striparray( $_, \@out);
  1194. @protocol= striparray( $_, \@protocol);
  1195. }
  1196. for my $strip (@strippart) {
  1197. chomp $strip;
  1198. for(@out) {
  1199. eval $strip;
  1200. }
  1201. }
  1202. if($hash{'crlf'}) {
  1203. subnewlines(1, \$_) for @protocol;
  1204. }
  1205. if((!$out[0] || ($out[0] eq "")) && $protocol[0]) {
  1206. logmsg "\n $testnum: protocol FAILED!\n".
  1207. " There was no content at all in the file $logdir/$SERVERIN.\n".
  1208. " Server glitch? Total curl failure? Returned: $cmdres\n";
  1209. # timestamp test result verification end
  1210. $timevrfyend{$testnum} = Time::HiRes::time();
  1211. return -1;
  1212. }
  1213. $res = compare($runnerid, $testnum, $testname, "protocol", \@out, \@protocol);
  1214. if($res) {
  1215. return -1;
  1216. }
  1217. $ok .= "p";
  1218. }
  1219. else {
  1220. $ok .= "-"; # protocol not checked
  1221. }
  1222. my %replyattr = getpartattr("reply", "data");
  1223. my @reply;
  1224. if (partexists("reply", "datacheck")) {
  1225. for my $partsuffix (('', '1', '2', '3', '4')) {
  1226. my @replycheckpart = getpart("reply", "datacheck".$partsuffix);
  1227. if(@replycheckpart) {
  1228. my %replycheckpartattr = getpartattr("reply", "datacheck".$partsuffix);
  1229. # get the mode attribute
  1230. my $filemode=$replycheckpartattr{'mode'};
  1231. if($filemode && ($filemode eq "text") && $has_textaware) {
  1232. # text mode when running on windows: fix line endings
  1233. s/\r\n/\n/g for @replycheckpart;
  1234. s/\n/\r\n/g for @replycheckpart;
  1235. }
  1236. if($replycheckpartattr{'nonewline'}) {
  1237. # Yes, we must cut off the final newline from the final line
  1238. # of the datacheck
  1239. chomp($replycheckpart[-1]);
  1240. }
  1241. if($replycheckpartattr{'crlf'} ||
  1242. ($feature{"hyper"} && ($keywords{"HTTP"}
  1243. || $keywords{"HTTPS"}))) {
  1244. subnewlines(0, \$_) for @replycheckpart;
  1245. }
  1246. push(@reply, @replycheckpart);
  1247. }
  1248. }
  1249. }
  1250. else {
  1251. # check against the data section
  1252. @reply = getpart("reply", "data");
  1253. if(@reply) {
  1254. if($replyattr{'nonewline'}) {
  1255. # cut off the final newline from the final line of the data
  1256. chomp($reply[-1]);
  1257. }
  1258. }
  1259. # get the mode attribute
  1260. my $filemode=$replyattr{'mode'};
  1261. if($filemode && ($filemode eq "text") && $has_textaware) {
  1262. # text mode when running on windows: fix line endings
  1263. s/\r\n/\n/g for @reply;
  1264. s/\n/\r\n/g for @reply;
  1265. }
  1266. if($replyattr{'crlf'} ||
  1267. ($feature{"hyper"} && ($keywords{"HTTP"}
  1268. || $keywords{"HTTPS"}))) {
  1269. subnewlines(0, \$_) for @reply;
  1270. }
  1271. }
  1272. if(!$replyattr{'nocheck'} && (@reply || $replyattr{'sendzero'})) {
  1273. # verify the received data
  1274. my @out = loadarray($CURLOUT);
  1275. $res = compare($runnerid, $testnum, $testname, "data", \@out, \@reply);
  1276. if ($res) {
  1277. return -1;
  1278. }
  1279. $ok .= "d";
  1280. }
  1281. else {
  1282. $ok .= "-"; # data not checked
  1283. }
  1284. # if this section exists, we verify upload
  1285. my @upload = getpart("verify", "upload");
  1286. if(@upload) {
  1287. my %hash = getpartattr("verify", "upload");
  1288. if($hash{'nonewline'}) {
  1289. # cut off the final newline from the final line of the upload data
  1290. chomp($upload[-1]);
  1291. }
  1292. # verify uploaded data
  1293. my @out = loadarray("$logdir/upload.$testnum");
  1294. for my $strip (@strippart) {
  1295. chomp $strip;
  1296. for(@out) {
  1297. eval $strip;
  1298. }
  1299. }
  1300. $res = compare($runnerid, $testnum, $testname, "upload", \@out, \@upload);
  1301. if ($res) {
  1302. return -1;
  1303. }
  1304. $ok .= "u";
  1305. }
  1306. else {
  1307. $ok .= "-"; # upload not checked
  1308. }
  1309. # this is the valid protocol blurb curl should generate to a proxy
  1310. my @proxyprot = getpart("verify", "proxy");
  1311. if(@proxyprot) {
  1312. # Verify the sent proxy request
  1313. # check if there's any attributes on the verify/protocol section
  1314. my %hash = getpartattr("verify", "proxy");
  1315. if($hash{'nonewline'}) {
  1316. # Yes, we must cut off the final newline from the final line
  1317. # of the protocol data
  1318. chomp($proxyprot[-1]);
  1319. }
  1320. my @out = loadarray("$logdir/$PROXYIN");
  1321. for(@strip) {
  1322. # strip off all lines that match the patterns from both arrays
  1323. chomp $_;
  1324. @out = striparray( $_, \@out);
  1325. @proxyprot= striparray( $_, \@proxyprot);
  1326. }
  1327. for my $strip (@strippart) {
  1328. chomp $strip;
  1329. for(@out) {
  1330. eval $strip;
  1331. }
  1332. }
  1333. if($hash{'crlf'} ||
  1334. ($feature{"hyper"} && ($keywords{"HTTP"} || $keywords{"HTTPS"}))) {
  1335. subnewlines(0, \$_) for @proxyprot;
  1336. }
  1337. $res = compare($runnerid, $testnum, $testname, "proxy", \@out, \@proxyprot);
  1338. if($res) {
  1339. return -1;
  1340. }
  1341. $ok .= "P";
  1342. }
  1343. else {
  1344. $ok .= "-"; # proxy not checked
  1345. }
  1346. my $outputok;
  1347. for my $partsuffix (('', '1', '2', '3', '4')) {
  1348. my @outfile=getpart("verify", "file".$partsuffix);
  1349. if(@outfile || partexists("verify", "file".$partsuffix) ) {
  1350. # we're supposed to verify a dynamically generated file!
  1351. my %hash = getpartattr("verify", "file".$partsuffix);
  1352. my $filename=$hash{'name'};
  1353. if(!$filename) {
  1354. logmsg " $testnum: IGNORED: section verify=>file$partsuffix ".
  1355. "has no name attribute\n";
  1356. if (runnerac_stopservers($runnerid)) {
  1357. logmsg "ERROR: runner $runnerid seems to have died\n";
  1358. } else {
  1359. # TODO: this is a blocking call that will stall the controller,
  1360. if($verbose) {
  1361. logmsg "WARNING: blocking call in async function\n";
  1362. }
  1363. # but this error condition should never happen except during
  1364. # development.
  1365. my ($rid, $unexpected, $logs) = runnerar($runnerid);
  1366. if(!$rid) {
  1367. logmsg "ERROR: runner $runnerid seems to have died\n";
  1368. } else {
  1369. logmsg $logs;
  1370. }
  1371. }
  1372. # timestamp test result verification end
  1373. $timevrfyend{$testnum} = Time::HiRes::time();
  1374. return -1;
  1375. }
  1376. my @generated=loadarray($filename);
  1377. # what parts to cut off from the file
  1378. my @stripfilepar = getpart("verify", "stripfile".$partsuffix);
  1379. my $filemode=$hash{'mode'};
  1380. if($filemode && ($filemode eq "text") && $has_textaware) {
  1381. # text mode when running on windows: fix line endings
  1382. s/\r\n/\n/g for @outfile;
  1383. s/\n/\r\n/g for @outfile;
  1384. }
  1385. if($hash{'crlf'} ||
  1386. ($feature{"hyper"} && ($keywords{"HTTP"}
  1387. || $keywords{"HTTPS"}))) {
  1388. subnewlines(0, \$_) for @outfile;
  1389. }
  1390. for my $strip (@stripfilepar) {
  1391. chomp $strip;
  1392. my @newgen;
  1393. for(@generated) {
  1394. eval $strip;
  1395. if($_) {
  1396. push @newgen, $_;
  1397. }
  1398. }
  1399. # this is to get rid of array entries that vanished (zero
  1400. # length) because of replacements
  1401. @generated = @newgen;
  1402. }
  1403. if($hash{'nonewline'}) {
  1404. # cut off the final newline from the final line of the
  1405. # output data
  1406. chomp($outfile[-1]);
  1407. }
  1408. $res = compare($runnerid, $testnum, $testname, "output ($filename)",
  1409. \@generated, \@outfile);
  1410. if($res) {
  1411. return -1;
  1412. }
  1413. $outputok = 1; # output checked
  1414. }
  1415. }
  1416. $ok .= ($outputok) ? "o" : "-"; # output checked or not
  1417. # verify SOCKS proxy details
  1418. my @socksprot = getpart("verify", "socks");
  1419. if(@socksprot) {
  1420. # Verify the sent SOCKS proxy details
  1421. my @out = loadarray("$logdir/$SOCKSIN");
  1422. $res = compare($runnerid, $testnum, $testname, "socks", \@out, \@socksprot);
  1423. if($res) {
  1424. return -1;
  1425. }
  1426. }
  1427. # accept multiple comma-separated error codes
  1428. my @splerr = split(/ *, */, $errorcode);
  1429. my $errok;
  1430. foreach my $e (@splerr) {
  1431. if($e == $cmdres) {
  1432. # a fine error code
  1433. $errok = 1;
  1434. last;
  1435. }
  1436. }
  1437. if($errok) {
  1438. $ok .= "e";
  1439. }
  1440. else {
  1441. if(!$short) {
  1442. logmsg sprintf("\n%s returned $cmdres, when expecting %s\n",
  1443. (!$tool)?"curl":$tool, $errorcode);
  1444. }
  1445. logmsg " $testnum: exit FAILED\n";
  1446. # timestamp test result verification end
  1447. $timevrfyend{$testnum} = Time::HiRes::time();
  1448. return -1;
  1449. }
  1450. if($feature{"TrackMemory"}) {
  1451. if(! -f "$logdir/$MEMDUMP") {
  1452. my %cmdhash = getpartattr("client", "command");
  1453. my $cmdtype = $cmdhash{'type'} || "default";
  1454. logmsg "\n** ALERT! memory tracking with no output file?\n"
  1455. if(!$cmdtype eq "perl");
  1456. $ok .= "-"; # problem with memory checking
  1457. }
  1458. else {
  1459. my @memdata=`$memanalyze "$logdir/$MEMDUMP"`;
  1460. my $leak=0;
  1461. for(@memdata) {
  1462. if($_ ne "") {
  1463. # well it could be other memory problems as well, but
  1464. # we call it leak for short here
  1465. $leak=1;
  1466. }
  1467. }
  1468. if($leak) {
  1469. logmsg "\n** MEMORY FAILURE\n";
  1470. logmsg @memdata;
  1471. # timestamp test result verification end
  1472. $timevrfyend{$testnum} = Time::HiRes::time();
  1473. return -1;
  1474. }
  1475. else {
  1476. $ok .= "m";
  1477. }
  1478. }
  1479. }
  1480. else {
  1481. $ok .= "-"; # memory not checked
  1482. }
  1483. if($valgrind) {
  1484. if($usedvalgrind) {
  1485. if(!opendir(DIR, "$logdir")) {
  1486. logmsg "ERROR: unable to read $logdir\n";
  1487. # timestamp test result verification end
  1488. $timevrfyend{$testnum} = Time::HiRes::time();
  1489. return -1;
  1490. }
  1491. my @files = readdir(DIR);
  1492. closedir(DIR);
  1493. my $vgfile;
  1494. foreach my $file (@files) {
  1495. if($file =~ /^valgrind$testnum(\..*|)$/) {
  1496. $vgfile = $file;
  1497. last;
  1498. }
  1499. }
  1500. if(!$vgfile) {
  1501. logmsg "ERROR: valgrind log file missing for test $testnum\n";
  1502. # timestamp test result verification end
  1503. $timevrfyend{$testnum} = Time::HiRes::time();
  1504. return -1;
  1505. }
  1506. my @e = valgrindparse("$logdir/$vgfile");
  1507. if(@e && $e[0]) {
  1508. if($automakestyle) {
  1509. logmsg "FAIL: $testnum - $testname - valgrind\n";
  1510. }
  1511. else {
  1512. logmsg " valgrind ERROR ";
  1513. logmsg @e;
  1514. }
  1515. # timestamp test result verification end
  1516. $timevrfyend{$testnum} = Time::HiRes::time();
  1517. return -1;
  1518. }
  1519. $ok .= "v";
  1520. }
  1521. else {
  1522. if($verbose) {
  1523. logmsg " valgrind SKIPPED\n";
  1524. }
  1525. $ok .= "-"; # skipped
  1526. }
  1527. }
  1528. else {
  1529. $ok .= "-"; # valgrind not checked
  1530. }
  1531. # add 'E' for event-based
  1532. $ok .= $run_event_based ? "E" : "-";
  1533. logmsg "$ok " if(!$short);
  1534. # timestamp test result verification end
  1535. $timevrfyend{$testnum} = Time::HiRes::time();
  1536. return 0;
  1537. }
  1538. #######################################################################
  1539. # Report a successful test
  1540. sub singletest_success {
  1541. my ($testnum, $count, $total, $errorreturncode)=@_;
  1542. my $sofar= time()-$start;
  1543. my $esttotal = $sofar/$count * $total;
  1544. my $estleft = $esttotal - $sofar;
  1545. my $timeleft=sprintf("remaining: %02d:%02d",
  1546. $estleft/60,
  1547. $estleft%60);
  1548. my $took = $timevrfyend{$testnum} - $timeprepini{$testnum};
  1549. my $duration = sprintf("duration: %02d:%02d",
  1550. $sofar/60, $sofar%60);
  1551. if(!$automakestyle) {
  1552. logmsg sprintf("OK (%-3d out of %-3d, %s, took %.3fs, %s)\n",
  1553. $count, $total, $timeleft, $took, $duration);
  1554. }
  1555. else {
  1556. my $testname= (getpart("client", "name"))[0];
  1557. chomp $testname;
  1558. logmsg "PASS: $testnum - $testname\n";
  1559. }
  1560. if($errorreturncode==2) {
  1561. logmsg "Warning: test$testnum result is ignored, but passed!\n";
  1562. }
  1563. }
  1564. #######################################################################
  1565. # Run a single specified test case
  1566. # This is structured as a state machine which changes state after an
  1567. # asynchronous call is made that awaits a response. The function returns with
  1568. # an error code and a flag that indicates if the state machine has completed,
  1569. # which means (if not) the function must be called again once the response has
  1570. # arrived.
  1571. #
  1572. sub singletest {
  1573. my ($runnerid, $testnum, $count, $total)=@_;
  1574. # start buffering logmsg; stop it on return
  1575. logmsg_bufferfortest($runnerid);
  1576. if(!exists $singletest_state{$runnerid}) {
  1577. # First time in singletest() for this test
  1578. $singletest_state{$runnerid} = ST_INIT;
  1579. }
  1580. if($singletest_state{$runnerid} == ST_INIT) {
  1581. my $logdir = getrunnerlogdir($runnerid);
  1582. # first, remove all lingering log & lock files
  1583. if((!cleardir($logdir) || !cleardir("$logdir/$LOCKDIR"))
  1584. && $clearlocks) {
  1585. # On Windows, lock files can't be deleted when the process still
  1586. # has them open, so kill those processes first
  1587. if(runnerac_clearlocks($runnerid, "$logdir/$LOCKDIR")) {
  1588. logmsg "ERROR: runner $runnerid seems to have died\n";
  1589. $singletest_state{$runnerid} = ST_INIT;
  1590. return (-1, 0);
  1591. }
  1592. $singletest_state{$runnerid} = ST_CLEARLOCKS;
  1593. } else {
  1594. $singletest_state{$runnerid} = ST_INITED;
  1595. # Recursively call the state machine again because there is no
  1596. # event expected that would otherwise trigger a new call.
  1597. return singletest(@_);
  1598. }
  1599. } elsif($singletest_state{$runnerid} == ST_CLEARLOCKS) {
  1600. my ($rid, $logs) = runnerar($runnerid);
  1601. if(!$rid) {
  1602. logmsg "ERROR: runner $runnerid seems to have died\n";
  1603. $singletest_state{$runnerid} = ST_INIT;
  1604. return (-1, 0);
  1605. }
  1606. logmsg $logs;
  1607. my $logdir = getrunnerlogdir($runnerid);
  1608. cleardir($logdir);
  1609. $singletest_state{$runnerid} = ST_INITED;
  1610. # Recursively call the state machine again because there is no
  1611. # event expected that would otherwise trigger a new call.
  1612. return singletest(@_);
  1613. } elsif($singletest_state{$runnerid} == ST_INITED) {
  1614. ###################################################################
  1615. # Restore environment variables that were modified in a previous run.
  1616. # Test definition may instruct to (un)set environment vars.
  1617. # This is done this early so that leftover variables don't affect
  1618. # starting servers or CI registration.
  1619. # restore_test_env(1);
  1620. ###################################################################
  1621. # Load test file so CI registration can get the right data before the
  1622. # runner is called
  1623. loadtest("${TESTDIR}/test${testnum}");
  1624. ###################################################################
  1625. # Register the test case with the CI environment
  1626. citest_starttest($testnum);
  1627. if(runnerac_test_preprocess($runnerid, $testnum)) {
  1628. logmsg "ERROR: runner $runnerid seems to have died\n";
  1629. $singletest_state{$runnerid} = ST_INIT;
  1630. return (-1, 0);
  1631. }
  1632. $singletest_state{$runnerid} = ST_PREPROCESS;
  1633. } elsif($singletest_state{$runnerid} == ST_PREPROCESS) {
  1634. my ($rid, $why, $error, $logs, $testtimings) = runnerar($runnerid);
  1635. if(!$rid) {
  1636. logmsg "ERROR: runner $runnerid seems to have died\n";
  1637. $singletest_state{$runnerid} = ST_INIT;
  1638. return (-1, 0);
  1639. }
  1640. logmsg $logs;
  1641. updatetesttimings($testnum, %$testtimings);
  1642. if($error == -2) {
  1643. if($postmortem) {
  1644. # Error indicates an actual problem starting the server, so
  1645. # display the server logs
  1646. displaylogs($rid, $testnum);
  1647. }
  1648. }
  1649. #######################################################################
  1650. # Load test file for this test number
  1651. my $logdir = getrunnerlogdir($runnerid);
  1652. loadtest("${logdir}/test${testnum}");
  1653. #######################################################################
  1654. # Print the test name and count tests
  1655. $error = singletest_count($testnum, $why);
  1656. if($error) {
  1657. # Submit the test case result with the CI environment
  1658. citest_finishtest($testnum, $error);
  1659. $singletest_state{$runnerid} = ST_INIT;
  1660. logmsg singletest_dumplogs();
  1661. return ($error, 0);
  1662. }
  1663. #######################################################################
  1664. # Execute this test number
  1665. my $cmdres;
  1666. my $CURLOUT;
  1667. my $tool;
  1668. my $usedvalgrind;
  1669. if(runnerac_test_run($runnerid, $testnum)) {
  1670. logmsg "ERROR: runner $runnerid seems to have died\n";
  1671. $singletest_state{$runnerid} = ST_INIT;
  1672. return (-1, 0);
  1673. }
  1674. $singletest_state{$runnerid} = ST_RUN;
  1675. } elsif($singletest_state{$runnerid} == ST_RUN) {
  1676. my ($rid, $error, $logs, $testtimings, $cmdres, $CURLOUT, $tool, $usedvalgrind) = runnerar($runnerid);
  1677. if(!$rid) {
  1678. logmsg "ERROR: runner $runnerid seems to have died\n";
  1679. $singletest_state{$runnerid} = ST_INIT;
  1680. return (-1, 0);
  1681. }
  1682. logmsg $logs;
  1683. updatetesttimings($testnum, %$testtimings);
  1684. if($error == -1) {
  1685. # no further verification will occur
  1686. $timevrfyend{$testnum} = Time::HiRes::time();
  1687. my $err = ignoreresultcode($testnum);
  1688. # Submit the test case result with the CI environment
  1689. citest_finishtest($testnum, $err);
  1690. $singletest_state{$runnerid} = ST_INIT;
  1691. logmsg singletest_dumplogs();
  1692. # return a test failure, either to be reported or to be ignored
  1693. return ($err, 0);
  1694. }
  1695. elsif($error == -2) {
  1696. # fill in the missing timings on error
  1697. timestampskippedevents($testnum);
  1698. # Submit the test case result with the CI environment
  1699. citest_finishtest($testnum, $error);
  1700. $singletest_state{$runnerid} = ST_INIT;
  1701. logmsg singletest_dumplogs();
  1702. return ($error, 0);
  1703. }
  1704. elsif($error > 0) {
  1705. # no further verification will occur
  1706. $timevrfyend{$testnum} = Time::HiRes::time();
  1707. # Submit the test case result with the CI environment
  1708. citest_finishtest($testnum, $error);
  1709. $singletest_state{$runnerid} = ST_INIT;
  1710. logmsg singletest_dumplogs();
  1711. return ($error, 0);
  1712. }
  1713. #######################################################################
  1714. # Verify that the test succeeded
  1715. #
  1716. # Load test file for this test number
  1717. my $logdir = getrunnerlogdir($runnerid);
  1718. loadtest("${logdir}/test${testnum}");
  1719. readtestkeywords();
  1720. $error = singletest_check($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind);
  1721. if($error == -1) {
  1722. my $err = ignoreresultcode($testnum);
  1723. # Submit the test case result with the CI environment
  1724. citest_finishtest($testnum, $err);
  1725. $singletest_state{$runnerid} = ST_INIT;
  1726. logmsg singletest_dumplogs();
  1727. # return a test failure, either to be reported or to be ignored
  1728. return ($err, 0);
  1729. }
  1730. elsif($error == -2) {
  1731. # torture test; there is no verification, so the run result holds the
  1732. # test success code
  1733. # Submit the test case result with the CI environment
  1734. citest_finishtest($testnum, $cmdres);
  1735. $singletest_state{$runnerid} = ST_INIT;
  1736. logmsg singletest_dumplogs();
  1737. return ($cmdres, 0);
  1738. }
  1739. #######################################################################
  1740. # Report a successful test
  1741. singletest_success($testnum, $count, $total, ignoreresultcode($testnum));
  1742. # Submit the test case result with the CI environment
  1743. citest_finishtest($testnum, 0);
  1744. $singletest_state{$runnerid} = ST_INIT;
  1745. logmsg singletest_dumplogs();
  1746. return (0, 0); # state machine is finished
  1747. }
  1748. singletest_unbufferlogs();
  1749. return (0, 1); # state machine must be called again on event
  1750. }
  1751. #######################################################################
  1752. # runtimestats displays test-suite run time statistics
  1753. #
  1754. sub runtimestats {
  1755. my $lasttest = $_[0];
  1756. return if(not $timestats);
  1757. logmsg "\nTest suite total running time breakdown per task...\n\n";
  1758. my @timesrvr;
  1759. my @timeprep;
  1760. my @timetool;
  1761. my @timelock;
  1762. my @timevrfy;
  1763. my @timetest;
  1764. my $timesrvrtot = 0.0;
  1765. my $timepreptot = 0.0;
  1766. my $timetooltot = 0.0;
  1767. my $timelocktot = 0.0;
  1768. my $timevrfytot = 0.0;
  1769. my $timetesttot = 0.0;
  1770. my $counter;
  1771. for my $testnum (1 .. $lasttest) {
  1772. if($timesrvrini{$testnum}) {
  1773. $timesrvrtot += $timesrvrend{$testnum} - $timesrvrini{$testnum};
  1774. $timepreptot +=
  1775. (($timetoolini{$testnum} - $timeprepini{$testnum}) -
  1776. ($timesrvrend{$testnum} - $timesrvrini{$testnum}));
  1777. $timetooltot += $timetoolend{$testnum} - $timetoolini{$testnum};
  1778. $timelocktot += $timesrvrlog{$testnum} - $timetoolend{$testnum};
  1779. $timevrfytot += $timevrfyend{$testnum} - $timesrvrlog{$testnum};
  1780. $timetesttot += $timevrfyend{$testnum} - $timeprepini{$testnum};
  1781. push @timesrvr, sprintf("%06.3f %04d",
  1782. $timesrvrend{$testnum} - $timesrvrini{$testnum}, $testnum);
  1783. push @timeprep, sprintf("%06.3f %04d",
  1784. ($timetoolini{$testnum} - $timeprepini{$testnum}) -
  1785. ($timesrvrend{$testnum} - $timesrvrini{$testnum}), $testnum);
  1786. push @timetool, sprintf("%06.3f %04d",
  1787. $timetoolend{$testnum} - $timetoolini{$testnum}, $testnum);
  1788. push @timelock, sprintf("%06.3f %04d",
  1789. $timesrvrlog{$testnum} - $timetoolend{$testnum}, $testnum);
  1790. push @timevrfy, sprintf("%06.3f %04d",
  1791. $timevrfyend{$testnum} - $timesrvrlog{$testnum}, $testnum);
  1792. push @timetest, sprintf("%06.3f %04d",
  1793. $timevrfyend{$testnum} - $timeprepini{$testnum}, $testnum);
  1794. }
  1795. }
  1796. {
  1797. no warnings 'numeric';
  1798. @timesrvr = sort { $b <=> $a } @timesrvr;
  1799. @timeprep = sort { $b <=> $a } @timeprep;
  1800. @timetool = sort { $b <=> $a } @timetool;
  1801. @timelock = sort { $b <=> $a } @timelock;
  1802. @timevrfy = sort { $b <=> $a } @timevrfy;
  1803. @timetest = sort { $b <=> $a } @timetest;
  1804. }
  1805. logmsg "Spent ". sprintf("%08.3f ", $timesrvrtot) .
  1806. "seconds starting and verifying test harness servers.\n";
  1807. logmsg "Spent ". sprintf("%08.3f ", $timepreptot) .
  1808. "seconds reading definitions and doing test preparations.\n";
  1809. logmsg "Spent ". sprintf("%08.3f ", $timetooltot) .
  1810. "seconds actually running test tools.\n";
  1811. logmsg "Spent ". sprintf("%08.3f ", $timelocktot) .
  1812. "seconds awaiting server logs lock removal.\n";
  1813. logmsg "Spent ". sprintf("%08.3f ", $timevrfytot) .
  1814. "seconds verifying test results.\n";
  1815. logmsg "Spent ". sprintf("%08.3f ", $timetesttot) .
  1816. "seconds doing all of the above.\n";
  1817. $counter = 25;
  1818. logmsg "\nTest server starting and verification time per test ".
  1819. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1820. logmsg "-time- test\n";
  1821. logmsg "------ ----\n";
  1822. foreach my $txt (@timesrvr) {
  1823. last if((not $fullstats) && (not $counter--));
  1824. logmsg "$txt\n";
  1825. }
  1826. $counter = 10;
  1827. logmsg "\nTest definition reading and preparation time per test ".
  1828. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1829. logmsg "-time- test\n";
  1830. logmsg "------ ----\n";
  1831. foreach my $txt (@timeprep) {
  1832. last if((not $fullstats) && (not $counter--));
  1833. logmsg "$txt\n";
  1834. }
  1835. $counter = 25;
  1836. logmsg "\nTest tool execution time per test ".
  1837. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1838. logmsg "-time- test\n";
  1839. logmsg "------ ----\n";
  1840. foreach my $txt (@timetool) {
  1841. last if((not $fullstats) && (not $counter--));
  1842. logmsg "$txt\n";
  1843. }
  1844. $counter = 15;
  1845. logmsg "\nTest server logs lock removal time per test ".
  1846. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1847. logmsg "-time- test\n";
  1848. logmsg "------ ----\n";
  1849. foreach my $txt (@timelock) {
  1850. last if((not $fullstats) && (not $counter--));
  1851. logmsg "$txt\n";
  1852. }
  1853. $counter = 10;
  1854. logmsg "\nTest results verification time per test ".
  1855. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1856. logmsg "-time- test\n";
  1857. logmsg "------ ----\n";
  1858. foreach my $txt (@timevrfy) {
  1859. last if((not $fullstats) && (not $counter--));
  1860. logmsg "$txt\n";
  1861. }
  1862. $counter = 50;
  1863. logmsg "\nTotal time per test ".
  1864. sprintf("(%s)...\n\n", (not $fullstats)?"top $counter":"full");
  1865. logmsg "-time- test\n";
  1866. logmsg "------ ----\n";
  1867. foreach my $txt (@timetest) {
  1868. last if((not $fullstats) && (not $counter--));
  1869. logmsg "$txt\n";
  1870. }
  1871. logmsg "\n";
  1872. }
  1873. #######################################################################
  1874. # returns code indicating why a test was skipped
  1875. # 0=unknown test, 1=use test result, 2=ignore test result
  1876. #
  1877. sub ignoreresultcode {
  1878. my ($testnum)=@_;
  1879. if(defined $ignoretestcodes{$testnum}) {
  1880. return $ignoretestcodes{$testnum};
  1881. }
  1882. return 0;
  1883. }
  1884. #######################################################################
  1885. # Put the given runner ID onto the queue of runners ready for a new task
  1886. #
  1887. sub runnerready {
  1888. my ($runnerid)=@_;
  1889. push @runnersidle, $runnerid;
  1890. }
  1891. #######################################################################
  1892. # Create test runners
  1893. #
  1894. sub createrunners {
  1895. my ($numrunners)=@_;
  1896. if(! $numrunners) {
  1897. $numrunners++;
  1898. }
  1899. # create $numrunners runners with minimum 1
  1900. for my $runnernum (1..$numrunners) {
  1901. my $dir = getrunnernumlogdir($runnernum);
  1902. cleardir($dir);
  1903. mkdir($dir, 0777);
  1904. $runnerids{$runnernum} = runner_init($dir, $jobs);
  1905. runnerready($runnerids{$runnernum});
  1906. }
  1907. }
  1908. #######################################################################
  1909. # Pick a test runner for the given test
  1910. #
  1911. sub pickrunner {
  1912. my ($testnum)=@_;
  1913. scalar(@runnersidle) || die "No runners available";
  1914. return pop @runnersidle;
  1915. }
  1916. #######################################################################
  1917. # Check options to this test program
  1918. #
  1919. # Special case for CMake: replace '$TFLAGS' by the contents of the
  1920. # environment variable (if any).
  1921. if(@ARGV && $ARGV[-1] eq '$TFLAGS') {
  1922. pop @ARGV;
  1923. push(@ARGV, split(' ', $ENV{'TFLAGS'})) if defined($ENV{'TFLAGS'});
  1924. }
  1925. $valgrind = checktestcmd("valgrind");
  1926. my $number=0;
  1927. my $fromnum=-1;
  1928. my @testthis;
  1929. while(@ARGV) {
  1930. if ($ARGV[0] eq "-v") {
  1931. # verbose output
  1932. $verbose=1;
  1933. }
  1934. elsif ($ARGV[0] eq "-c") {
  1935. # use this path to curl instead of default
  1936. $DBGCURL=$CURL=$ARGV[1];
  1937. shift @ARGV;
  1938. }
  1939. elsif ($ARGV[0] eq "-vc") {
  1940. # use this path to a curl used to verify servers
  1941. # Particularly useful when you introduce a crashing bug somewhere in
  1942. # the development version as then it won't be able to run any tests
  1943. # since it can't verify the servers!
  1944. $VCURL=shell_quote($ARGV[1]);
  1945. shift @ARGV;
  1946. }
  1947. elsif ($ARGV[0] eq "-ac") {
  1948. # use this curl only to talk to APIs (currently only CI test APIs)
  1949. $ACURL=shell_quote($ARGV[1]);
  1950. shift @ARGV;
  1951. }
  1952. elsif ($ARGV[0] eq "-d") {
  1953. # have the servers display protocol output
  1954. $debugprotocol=1;
  1955. }
  1956. elsif($ARGV[0] eq "-e") {
  1957. # run the tests cases event based if possible
  1958. $run_event_based=1;
  1959. }
  1960. elsif($ARGV[0] eq "-f") {
  1961. # force - run the test case even if listed in DISABLED
  1962. $run_disabled=1;
  1963. }
  1964. elsif($ARGV[0] eq "-E") {
  1965. # load additional reasons to skip tests
  1966. shift @ARGV;
  1967. my $exclude_file = $ARGV[0];
  1968. open(my $fd, "<", $exclude_file) or die "Couldn't open '$exclude_file': $!";
  1969. while(my $line = <$fd>) {
  1970. next if ($line =~ /^#/);
  1971. chomp $line;
  1972. my ($type, $patterns, $skip_reason) = split(/\s*:\s*/, $line, 3);
  1973. die "Unsupported type: $type\n" if($type !~ /^keyword|test|tool$/);
  1974. foreach my $pattern (split(/,/, $patterns)) {
  1975. if($type eq "test") {
  1976. # Strip leading zeros in the test number
  1977. $pattern = int($pattern);
  1978. }
  1979. $custom_skip_reasons{$type}{$pattern} = $skip_reason;
  1980. }
  1981. }
  1982. close($fd);
  1983. }
  1984. elsif ($ARGV[0] eq "-g") {
  1985. # run this test with gdb
  1986. $gdbthis=1;
  1987. }
  1988. elsif ($ARGV[0] eq "-gl") {
  1989. # run this test with lldb
  1990. $gdbthis=2;
  1991. }
  1992. elsif ($ARGV[0] eq "-gw") {
  1993. # run this test with windowed gdb
  1994. $gdbthis=1;
  1995. $gdbxwin=1;
  1996. }
  1997. elsif($ARGV[0] eq "-s") {
  1998. # short output
  1999. $short=1;
  2000. }
  2001. elsif($ARGV[0] eq "-am") {
  2002. # automake-style output
  2003. $short=1;
  2004. $automakestyle=1;
  2005. }
  2006. elsif($ARGV[0] eq "-n") {
  2007. # no valgrind
  2008. undef $valgrind;
  2009. }
  2010. elsif($ARGV[0] eq "--no-debuginfod") {
  2011. # disable the valgrind debuginfod functionality
  2012. $no_debuginfod = 1;
  2013. }
  2014. elsif ($ARGV[0] eq "-R") {
  2015. # execute in scrambled order
  2016. $scrambleorder=1;
  2017. }
  2018. elsif($ARGV[0] =~ /^-t(.*)/) {
  2019. # torture
  2020. $torture=1;
  2021. my $xtra = $1;
  2022. if($xtra =~ s/(\d+)$//) {
  2023. $tortalloc = $1;
  2024. }
  2025. }
  2026. elsif($ARGV[0] =~ /--shallow=(\d+)/) {
  2027. # Fail no more than this amount per tests when running
  2028. # torture.
  2029. my ($num)=($1);
  2030. $shallow=$num;
  2031. }
  2032. elsif($ARGV[0] =~ /--repeat=(\d+)/) {
  2033. # Repeat-run the given tests this many times
  2034. $repeat = $1;
  2035. }
  2036. elsif($ARGV[0] =~ /--seed=(\d+)/) {
  2037. # Set a fixed random seed (used for -R and --shallow)
  2038. $randseed = $1;
  2039. }
  2040. elsif($ARGV[0] eq "-a") {
  2041. # continue anyway, even if a test fail
  2042. $anyway=1;
  2043. }
  2044. elsif($ARGV[0] eq "-o") {
  2045. shift @ARGV;
  2046. if ($ARGV[0] =~ /^(\w+)=([\w.:\/\[\]-]+)$/) {
  2047. my ($variable, $value) = ($1, $2);
  2048. eval "\$$variable='$value'" or die "Failed to set \$$variable to $value: $@";
  2049. } else {
  2050. die "Failed to parse '-o $ARGV[0]'. May contain unexpected characters.\n";
  2051. }
  2052. }
  2053. elsif($ARGV[0] eq "-p") {
  2054. $postmortem=1;
  2055. }
  2056. elsif($ARGV[0] eq "-P") {
  2057. shift @ARGV;
  2058. $proxy_address=$ARGV[0];
  2059. }
  2060. elsif($ARGV[0] eq "-L") {
  2061. # require additional library file
  2062. shift @ARGV;
  2063. require $ARGV[0];
  2064. }
  2065. elsif($ARGV[0] eq "-l") {
  2066. # lists the test case names only
  2067. $listonly=1;
  2068. }
  2069. elsif($ARGV[0] =~ /^-j(.*)/) {
  2070. # parallel jobs
  2071. $jobs=1;
  2072. my $xtra = $1;
  2073. if($xtra =~ s/(\d+)$//) {
  2074. $jobs = $1;
  2075. }
  2076. }
  2077. elsif($ARGV[0] eq "-k") {
  2078. # keep stdout and stderr files after tests
  2079. $keepoutfiles=1;
  2080. }
  2081. elsif($ARGV[0] eq "-r") {
  2082. # run time statistics needs Time::HiRes
  2083. if($Time::HiRes::VERSION) {
  2084. # presize hashes appropriately to hold an entire test run
  2085. keys(%timeprepini) = 2000;
  2086. keys(%timesrvrini) = 2000;
  2087. keys(%timesrvrend) = 2000;
  2088. keys(%timetoolini) = 2000;
  2089. keys(%timetoolend) = 2000;
  2090. keys(%timesrvrlog) = 2000;
  2091. keys(%timevrfyend) = 2000;
  2092. $timestats=1;
  2093. $fullstats=0;
  2094. }
  2095. }
  2096. elsif($ARGV[0] eq "-rf") {
  2097. # run time statistics needs Time::HiRes
  2098. if($Time::HiRes::VERSION) {
  2099. # presize hashes appropriately to hold an entire test run
  2100. keys(%timeprepini) = 2000;
  2101. keys(%timesrvrini) = 2000;
  2102. keys(%timesrvrend) = 2000;
  2103. keys(%timetoolini) = 2000;
  2104. keys(%timetoolend) = 2000;
  2105. keys(%timesrvrlog) = 2000;
  2106. keys(%timevrfyend) = 2000;
  2107. $timestats=1;
  2108. $fullstats=1;
  2109. }
  2110. }
  2111. elsif($ARGV[0] eq "-rm") {
  2112. # force removal of files by killing locking processes
  2113. $clearlocks=1;
  2114. }
  2115. elsif($ARGV[0] eq "-u") {
  2116. # error instead of warning on server unexpectedly alive
  2117. $err_unexpected=1;
  2118. }
  2119. elsif(($ARGV[0] eq "-h") || ($ARGV[0] eq "--help")) {
  2120. # show help text
  2121. print <<"EOHELP"
  2122. Usage: runtests.pl [options] [test selection(s)]
  2123. -a continue even if a test fails
  2124. -ac path use this curl only to talk to APIs (currently only CI test APIs)
  2125. -am automake style output PASS/FAIL: [number] [name]
  2126. -c path use this curl executable
  2127. -d display server debug info
  2128. -e event-based execution
  2129. -E file load the specified file to exclude certain tests
  2130. -f forcibly run even if disabled
  2131. -g run the test case with gdb
  2132. -gw run the test case with gdb as a windowed application
  2133. -h this help text
  2134. -j[N] spawn this number of processes to run tests (default 0)
  2135. -k keep stdout and stderr files present after tests
  2136. -L path require an additional perl library file to replace certain functions
  2137. -l list all test case names/descriptions
  2138. -n no valgrind
  2139. --no-debuginfod disable the valgrind debuginfod functionality
  2140. -o variable=value set internal variable to the specified value
  2141. -P proxy use the specified proxy
  2142. -p print log file contents when a test fails
  2143. -R scrambled order (uses the random seed, see --seed)
  2144. -r run time statistics
  2145. -rf full run time statistics
  2146. -rm force removal of files by killing locking processes (Windows only)
  2147. --repeat=[num] run the given tests this many times
  2148. -s short output
  2149. --seed=[num] set the random seed to a fixed number
  2150. --shallow=[num] randomly makes the torture tests "thinner"
  2151. -t[N] torture (simulate function failures); N means fail Nth function
  2152. -u error instead of warning on server unexpectedly alive
  2153. -v verbose output
  2154. -vc path use this curl only to verify the existing servers
  2155. [num] like "5 6 9" or " 5 to 22 " to run those tests only
  2156. [!num] like "!5 !6 !9" to disable those tests
  2157. [~num] like "~5 ~6 ~9" to ignore the result of those tests
  2158. [keyword] like "IPv6" to select only tests containing the key word
  2159. [!keyword] like "!cookies" to disable any tests containing the key word
  2160. [~keyword] like "~cookies" to ignore results of tests containing key word
  2161. EOHELP
  2162. ;
  2163. exit;
  2164. }
  2165. elsif($ARGV[0] =~ /^(\d+)/) {
  2166. $number = $1;
  2167. if($fromnum >= 0) {
  2168. for my $n ($fromnum .. $number) {
  2169. push @testthis, $n;
  2170. }
  2171. $fromnum = -1;
  2172. }
  2173. else {
  2174. push @testthis, $1;
  2175. }
  2176. }
  2177. elsif($ARGV[0] =~ /^to$/i) {
  2178. $fromnum = $number+1;
  2179. }
  2180. elsif($ARGV[0] =~ /^!(\d+)/) {
  2181. $fromnum = -1;
  2182. $disabled{$1}=$1;
  2183. }
  2184. elsif($ARGV[0] =~ /^~(\d+)/) {
  2185. $fromnum = -1;
  2186. $ignored{$1}=$1;
  2187. }
  2188. elsif($ARGV[0] =~ /^!(.+)/) {
  2189. $disabled_keywords{lc($1)}=$1;
  2190. }
  2191. elsif($ARGV[0] =~ /^~(.+)/) {
  2192. $ignored_keywords{lc($1)}=$1;
  2193. }
  2194. elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
  2195. $enabled_keywords{lc($1)}=$1;
  2196. }
  2197. else {
  2198. print "Unknown option: $ARGV[0]\n";
  2199. exit;
  2200. }
  2201. shift @ARGV;
  2202. }
  2203. delete $ENV{'DEBUGINFOD_URLS'} if($ENV{'DEBUGINFOD_URLS'} && $no_debuginfod);
  2204. if(!$randseed) {
  2205. my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
  2206. localtime(time);
  2207. # seed of the month. December 2019 becomes 201912
  2208. $randseed = ($year+1900)*100 + $mon+1;
  2209. print "Using curl: $CURL\n";
  2210. open(my $curlvh, "-|", shell_quote($CURL) . " --version 2>/dev/null") ||
  2211. die "could not get curl version!";
  2212. my @c = <$curlvh>;
  2213. close($curlvh) || die "could not get curl version!";
  2214. # use the first line of output and get the md5 out of it
  2215. my $str = md5($c[0]);
  2216. $randseed += unpack('S', $str); # unsigned 16 bit value
  2217. }
  2218. srand $randseed;
  2219. if(@testthis && ($testthis[0] ne "")) {
  2220. $TESTCASES=join(" ", @testthis);
  2221. }
  2222. if($valgrind) {
  2223. # we have found valgrind on the host, use it
  2224. # verify that we can invoke it fine
  2225. my $code = runclient("valgrind >/dev/null 2>&1");
  2226. if(($code>>8) != 1) {
  2227. #logmsg "Valgrind failure, disable it\n";
  2228. undef $valgrind;
  2229. } else {
  2230. # since valgrind 2.1.x, '--tool' option is mandatory
  2231. # use it, if it is supported by the version installed on the system
  2232. # (this happened in 2003, so we could probably don't need to care about
  2233. # that old version any longer and just delete this check)
  2234. runclient("valgrind --help 2>&1 | grep -- --tool > /dev/null 2>&1");
  2235. if (($? >> 8)) {
  2236. $valgrind_tool="";
  2237. }
  2238. open(my $curlh, "<", "$CURL");
  2239. my $l = <$curlh>;
  2240. if($l =~ /^\#\!/) {
  2241. # A shell script. This is typically when built with libtool,
  2242. $valgrind="../libtool --mode=execute $valgrind";
  2243. }
  2244. close($curlh);
  2245. # valgrind 3 renamed the --logfile option to --log-file!!!
  2246. # (this happened in 2005, so we could probably don't need to care about
  2247. # that old version any longer and just delete this check)
  2248. my $ver=join(' ', runclientoutput("valgrind --version"));
  2249. # cut off all but digits and dots
  2250. $ver =~ s/[^0-9.]//g;
  2251. if($ver =~ /^(\d+)/) {
  2252. $ver = $1;
  2253. if($ver < 3) {
  2254. $valgrind_logfile="--logfile";
  2255. }
  2256. }
  2257. }
  2258. }
  2259. if ($gdbthis) {
  2260. # open the executable curl and read the first 4 bytes of it
  2261. open(my $check, "<", "$CURL");
  2262. my $c;
  2263. sysread $check, $c, 4;
  2264. close($check);
  2265. if($c eq "#! /") {
  2266. # A shell script. This is typically when built with libtool,
  2267. $libtool = 1;
  2268. $gdb = "../libtool --mode=execute gdb";
  2269. }
  2270. }
  2271. #######################################################################
  2272. # clear and create logging directory:
  2273. #
  2274. # TODO: figure how to get around this. This dir is needed for checksystemfeatures()
  2275. # Maybe create & use & delete a temporary directory in that function
  2276. cleardir($LOGDIR);
  2277. mkdir($LOGDIR, 0777);
  2278. mkdir("$LOGDIR/$LOCKDIR", 0777);
  2279. #######################################################################
  2280. # initialize some variables
  2281. #
  2282. get_disttests();
  2283. if(!$jobs) {
  2284. # Disable buffered logging with only one test job
  2285. setlogfunc(\&logmsg);
  2286. }
  2287. #######################################################################
  2288. # Output curl version and host info being tested
  2289. #
  2290. if(!$listonly) {
  2291. checksystemfeatures();
  2292. }
  2293. #######################################################################
  2294. # initialize configuration needed to set up servers
  2295. # TODO: rearrange things so this can be called only in runner_init()
  2296. #
  2297. initserverconfig();
  2298. if(!$listonly) {
  2299. # these can only be displayed after initserverconfig() has been called
  2300. displayserverfeatures();
  2301. # globally disabled tests
  2302. disabledtests("$TESTDIR/DISABLED");
  2303. }
  2304. #######################################################################
  2305. # Fetch all disabled tests, if there are any
  2306. #
  2307. sub disabledtests {
  2308. my ($file) = @_;
  2309. my @input;
  2310. if(open(my $disabledh, "<", "$file")) {
  2311. while(<$disabledh>) {
  2312. if(/^ *\#/) {
  2313. # allow comments
  2314. next;
  2315. }
  2316. push @input, $_;
  2317. }
  2318. close($disabledh);
  2319. # preprocess the input to make conditionally disabled tests depending
  2320. # on variables
  2321. my @pp = prepro(0, @input);
  2322. for my $t (@pp) {
  2323. if($t =~ /(\d+)/) {
  2324. my ($n) = $1;
  2325. $disabled{$n}=$n; # disable this test number
  2326. if(! -f "$srcdir/data/test$n") {
  2327. print STDERR "WARNING! Non-existing test $n in $file!\n";
  2328. # fail hard to make user notice
  2329. exit 1;
  2330. }
  2331. logmsg "DISABLED: test $n\n" if ($verbose);
  2332. }
  2333. else {
  2334. print STDERR "$file: rubbish content: $t\n";
  2335. exit 2;
  2336. }
  2337. }
  2338. }
  2339. }
  2340. #######################################################################
  2341. # If 'all' tests are requested, find out all test numbers
  2342. #
  2343. if ( $TESTCASES eq "all") {
  2344. # Get all commands and find out their test numbers
  2345. opendir(DIR, $TESTDIR) || die "can't opendir $TESTDIR: $!";
  2346. my @cmds = grep { /^test([0-9]+)$/ && -f "$TESTDIR/$_" } readdir(DIR);
  2347. closedir(DIR);
  2348. $TESTCASES=""; # start with no test cases
  2349. # cut off everything but the digits
  2350. for(@cmds) {
  2351. $_ =~ s/[a-z\/\.]*//g;
  2352. }
  2353. # sort the numbers from low to high
  2354. foreach my $n (sort { $a <=> $b } @cmds) {
  2355. if($disabled{$n}) {
  2356. # skip disabled test cases
  2357. my $why = "configured as DISABLED";
  2358. $skipped{$why}++;
  2359. $teststat[$n]=$why; # store reason for this test case
  2360. next;
  2361. }
  2362. $TESTCASES .= " $n";
  2363. }
  2364. }
  2365. else {
  2366. my $verified="";
  2367. for(split(" ", $TESTCASES)) {
  2368. if (-e "$TESTDIR/test$_") {
  2369. $verified.="$_ ";
  2370. }
  2371. }
  2372. if($verified eq "") {
  2373. print "No existing test cases were specified\n";
  2374. exit;
  2375. }
  2376. $TESTCASES = $verified;
  2377. }
  2378. if($repeat) {
  2379. my $s;
  2380. for(1 .. $repeat) {
  2381. $s .= $TESTCASES;
  2382. }
  2383. $TESTCASES = $s;
  2384. }
  2385. if($scrambleorder) {
  2386. # scramble the order of the test cases
  2387. my @rand;
  2388. while($TESTCASES) {
  2389. my @all = split(/ +/, $TESTCASES);
  2390. if(!$all[0]) {
  2391. # if the first is blank, shift away it
  2392. shift @all;
  2393. }
  2394. my $r = rand @all;
  2395. push @rand, $all[$r];
  2396. $all[$r]="";
  2397. $TESTCASES = join(" ", @all);
  2398. }
  2399. $TESTCASES = join(" ", @rand);
  2400. }
  2401. # Display the contents of the given file. Line endings are canonicalized
  2402. # and excessively long files are elided
  2403. sub displaylogcontent {
  2404. my ($file)=@_;
  2405. if(open(my $single, "<", "$file")) {
  2406. my $linecount = 0;
  2407. my $truncate;
  2408. my @tail;
  2409. while(my $string = <$single>) {
  2410. $string =~ s/\r\n/\n/g;
  2411. $string =~ s/[\r\f\032]/\n/g;
  2412. $string .= "\n" unless ($string =~ /\n$/);
  2413. $string =~ tr/\n//;
  2414. for my $line (split(m/\n/, $string)) {
  2415. $line =~ s/\s*\!$//;
  2416. if ($truncate) {
  2417. push @tail, " $line\n";
  2418. } else {
  2419. logmsg " $line\n";
  2420. }
  2421. $linecount++;
  2422. $truncate = $linecount > 1200;
  2423. }
  2424. }
  2425. close($single);
  2426. if(@tail) {
  2427. my $tailshow = 200;
  2428. my $tailskip = 0;
  2429. my $tailtotal = scalar @tail;
  2430. if($tailtotal > $tailshow) {
  2431. $tailskip = $tailtotal - $tailshow;
  2432. logmsg "=== File too long: $tailskip lines omitted here\n";
  2433. }
  2434. for($tailskip .. $tailtotal-1) {
  2435. logmsg "$tail[$_]";
  2436. }
  2437. }
  2438. }
  2439. }
  2440. sub displaylogs {
  2441. my ($runnerid, $testnum)=@_;
  2442. my $logdir = getrunnerlogdir($runnerid);
  2443. opendir(DIR, "$logdir") ||
  2444. die "can't open dir: $!";
  2445. my @logs = readdir(DIR);
  2446. closedir(DIR);
  2447. logmsg "== Contents of files in the $logdir/ dir after test $testnum\n";
  2448. foreach my $log (sort @logs) {
  2449. if($log =~ /\.(\.|)$/) {
  2450. next; # skip "." and ".."
  2451. }
  2452. if($log =~ /^\.nfs/) {
  2453. next; # skip ".nfs"
  2454. }
  2455. if(($log eq "memdump") || ($log eq "core")) {
  2456. next; # skip "memdump" and "core"
  2457. }
  2458. if((-d "$logdir/$log") || (! -s "$logdir/$log")) {
  2459. next; # skip directory and empty files
  2460. }
  2461. if(($log =~ /^stdout\d+/) && ($log !~ /^stdout$testnum/)) {
  2462. next; # skip stdoutNnn of other tests
  2463. }
  2464. if(($log =~ /^stderr\d+/) && ($log !~ /^stderr$testnum/)) {
  2465. next; # skip stderrNnn of other tests
  2466. }
  2467. if(($log =~ /^upload\d+/) && ($log !~ /^upload$testnum/)) {
  2468. next; # skip uploadNnn of other tests
  2469. }
  2470. if(($log =~ /^curl\d+\.out/) && ($log !~ /^curl$testnum\.out/)) {
  2471. next; # skip curlNnn.out of other tests
  2472. }
  2473. if(($log =~ /^test\d+\.txt/) && ($log !~ /^test$testnum\.txt/)) {
  2474. next; # skip testNnn.txt of other tests
  2475. }
  2476. if(($log =~ /^file\d+\.txt/) && ($log !~ /^file$testnum\.txt/)) {
  2477. next; # skip fileNnn.txt of other tests
  2478. }
  2479. if(($log =~ /^netrc\d+/) && ($log !~ /^netrc$testnum/)) {
  2480. next; # skip netrcNnn of other tests
  2481. }
  2482. if(($log =~ /^trace\d+/) && ($log !~ /^trace$testnum/)) {
  2483. next; # skip traceNnn of other tests
  2484. }
  2485. if(($log =~ /^valgrind\d+/) && ($log !~ /^valgrind$testnum(?:\..*)?$/)) {
  2486. next; # skip valgrindNnn of other tests
  2487. }
  2488. if(($log =~ /^test$testnum$/)) {
  2489. next; # skip test$testnum since it can be very big
  2490. }
  2491. logmsg "=== Start of file $log\n";
  2492. displaylogcontent("$logdir/$log");
  2493. logmsg "=== End of file $log\n";
  2494. }
  2495. }
  2496. #######################################################################
  2497. # Scan tests to find suitable candidates
  2498. #
  2499. my $failed;
  2500. my $failedign;
  2501. my $ok=0;
  2502. my $ign=0;
  2503. my $total=0;
  2504. my $lasttest=0;
  2505. my @at = split(" ", $TESTCASES);
  2506. my $count=0;
  2507. my $endwaitcnt=0;
  2508. $start = time();
  2509. # scan all tests to find ones we should try to run
  2510. my @runtests;
  2511. foreach my $testnum (@at) {
  2512. $lasttest = $testnum if($testnum > $lasttest);
  2513. my ($why, $errorreturncode) = singletest_shouldrun($testnum);
  2514. if($why || $listonly) {
  2515. # Display test name now--test will be completely skipped later
  2516. my $error = singletest_count($testnum, $why);
  2517. next;
  2518. }
  2519. $ignoretestcodes{$testnum} = $errorreturncode;
  2520. push(@runtests, $testnum);
  2521. }
  2522. my $totaltests = scalar(@runtests);
  2523. if($listonly) {
  2524. exit(0);
  2525. }
  2526. #######################################################################
  2527. # Setup CI Test Run
  2528. citest_starttestrun();
  2529. #######################################################################
  2530. # Start test runners
  2531. #
  2532. my $numrunners = $jobs < scalar(@runtests) ? $jobs : scalar(@runtests);
  2533. createrunners($numrunners);
  2534. #######################################################################
  2535. # The main test-loop
  2536. #
  2537. # Every iteration through the loop consists of these steps:
  2538. # - if the global abort flag is set, exit the loop; we are done
  2539. # - if a runner is idle, start a new test on it
  2540. # - if all runners are idle, exit the loop; we are done
  2541. # - if a runner has a response for us, process the response
  2542. # run through each candidate test and execute it
  2543. while () {
  2544. # check the abort flag
  2545. if($globalabort) {
  2546. logmsg singletest_dumplogs();
  2547. logmsg "Aborting tests\n";
  2548. logmsg "Waiting for " . scalar((keys %runnersrunning)) . " outstanding test(s) to finish...\n";
  2549. # Wait for the last requests to complete and throw them away so
  2550. # that IPC calls & responses stay in sync
  2551. # TODO: send a signal to the runners to interrupt a long test
  2552. foreach my $rid (keys %runnersrunning) {
  2553. runnerar($rid);
  2554. delete $runnersrunning{$rid};
  2555. logmsg ".";
  2556. $| = 1;
  2557. }
  2558. logmsg "\n";
  2559. last;
  2560. }
  2561. # Start a new test if possible
  2562. if(scalar(@runnersidle) && scalar(@runtests)) {
  2563. # A runner is ready to run a test, and tests are still available to run
  2564. # so start a new test.
  2565. $count++;
  2566. my $testnum = shift(@runtests);
  2567. # pick a runner for this new test
  2568. my $runnerid = pickrunner($testnum);
  2569. $countforrunner{$runnerid} = $count;
  2570. # Start the test
  2571. my ($error, $again) = singletest($runnerid, $testnum, $countforrunner{$runnerid}, $totaltests);
  2572. if($again) {
  2573. # this runner is busy running a test
  2574. $runnersrunning{$runnerid} = $testnum;
  2575. } else {
  2576. runnerready($runnerid);
  2577. if($error >= 0) {
  2578. # We make this simplifying assumption to avoid having to handle
  2579. # $error properly here, but we must handle the case of runner
  2580. # death without abending here.
  2581. die "Internal error: test must not complete on first call";
  2582. }
  2583. }
  2584. }
  2585. # See if we've completed all the tests
  2586. if(!scalar(%runnersrunning)) {
  2587. # No runners are running; we must be done
  2588. scalar(@runtests) && die 'Internal error: still have tests to run';
  2589. last;
  2590. }
  2591. # See if a test runner needs attention
  2592. # If we could be running more tests, don't wait so we can schedule a new
  2593. # one immediately. If all runners are busy, wait a fraction of a second
  2594. # for one to finish so we can still loop around to check the abort flag.
  2595. my $runnerwait = scalar(@runnersidle) && scalar(@runtests) ? 0 : 0.5;
  2596. my ($ridready, $riderror) = runnerar_ready($runnerwait);
  2597. if($ridready && ! defined $runnersrunning{$ridready}) {
  2598. # On Linux, a closed pipe still shows up as ready instead of error.
  2599. # Detect this here by seeing if we are expecting it to be ready and
  2600. # treat it as an error if not.
  2601. logmsg "ERROR: Runner $ridready is unexpectedly ready; is probably actually dead\n";
  2602. $riderror = $ridready;
  2603. undef $ridready;
  2604. }
  2605. if($ridready) {
  2606. # This runner is ready to be serviced
  2607. my $testnum = $runnersrunning{$ridready};
  2608. defined $testnum || die "Internal error: test for runner $ridready unknown";
  2609. delete $runnersrunning{$ridready};
  2610. my ($error, $again) = singletest($ridready, $testnum, $countforrunner{$ridready}, $totaltests);
  2611. if($again) {
  2612. # this runner is busy running a test
  2613. $runnersrunning{$ridready} = $testnum;
  2614. } else {
  2615. # Test is complete
  2616. runnerready($ridready);
  2617. if($error < 0) {
  2618. # not a test we can run
  2619. next;
  2620. }
  2621. $total++; # number of tests we've run
  2622. if($error>0) {
  2623. if($error==2) {
  2624. # ignored test failures
  2625. $failedign .= "$testnum ";
  2626. }
  2627. else {
  2628. $failed.= "$testnum ";
  2629. }
  2630. if($postmortem) {
  2631. # display all files in $LOGDIR/ in a nice way
  2632. displaylogs($ridready, $testnum);
  2633. }
  2634. if($error==2) {
  2635. $ign++; # ignored test result counter
  2636. }
  2637. elsif(!$anyway) {
  2638. # a test failed, abort
  2639. logmsg "\n - abort tests\n";
  2640. undef @runtests; # empty out the remaining tests
  2641. }
  2642. }
  2643. elsif(!$error) {
  2644. $ok++; # successful test counter
  2645. }
  2646. }
  2647. }
  2648. if($riderror) {
  2649. logmsg "ERROR: runner $riderror is dead! aborting test run\n";
  2650. delete $runnersrunning{$riderror} if(defined $runnersrunning{$riderror});
  2651. $globalabort = 1;
  2652. }
  2653. if(!scalar(@runtests) && ++$endwaitcnt == (240 + $jobs)) {
  2654. # Once all tests have been scheduled on a runner at the end of a test
  2655. # run, we just wait for their results to come in. If we're still
  2656. # waiting after a couple of minutes ($endwaitcnt multiplied by
  2657. # $runnerwait, plus $jobs because that number won't time out), display
  2658. # the same test runner status as we give with a SIGUSR1. This will
  2659. # likely point to a single test that has hung.
  2660. logmsg "Hmmm, the tests are taking a while to finish. Here is the status:\n";
  2661. catch_usr1();
  2662. }
  2663. }
  2664. my $sofar = time() - $start;
  2665. #######################################################################
  2666. # Finish CI Test Run
  2667. citest_finishtestrun();
  2668. # Tests done, stop the servers
  2669. foreach my $runnerid (values %runnerids) {
  2670. runnerac_stopservers($runnerid);
  2671. }
  2672. # Wait for servers to stop
  2673. my $unexpected;
  2674. foreach my $runnerid (values %runnerids) {
  2675. my ($rid, $unexpect, $logs) = runnerar($runnerid);
  2676. $unexpected ||= $unexpect;
  2677. logmsg $logs;
  2678. }
  2679. # Kill the runners
  2680. # There is a race condition here since we don't know exactly when the runners
  2681. # have each finished shutting themselves down, but we're about to exit so it
  2682. # doesn't make much difference.
  2683. foreach my $runnerid (values %runnerids) {
  2684. runnerac_shutdown($runnerid);
  2685. sleep 0; # give runner a context switch so it can shut itself down
  2686. }
  2687. my $numskipped = %skipped ? sum values %skipped : 0;
  2688. my $all = $total + $numskipped;
  2689. runtimestats($lasttest);
  2690. if($all) {
  2691. logmsg "TESTDONE: $all tests were considered during ".
  2692. sprintf("%.0f", $sofar) ." seconds.\n";
  2693. }
  2694. if(%skipped && !$short) {
  2695. my $s=0;
  2696. # Temporary hash to print the restraints sorted by the number
  2697. # of their occurrences
  2698. my %restraints;
  2699. logmsg "TESTINFO: $numskipped tests were skipped due to these restraints:\n";
  2700. for(keys %skipped) {
  2701. my $r = $_;
  2702. my $skip_count = $skipped{$r};
  2703. my $log_line = sprintf("TESTINFO: \"%s\" %d time%s (", $r, $skip_count,
  2704. ($skip_count == 1) ? "" : "s");
  2705. # now gather all test case numbers that had this reason for being
  2706. # skipped
  2707. my $c=0;
  2708. my $max = 9;
  2709. for(0 .. scalar @teststat) {
  2710. my $t = $_;
  2711. if($teststat[$t] && ($teststat[$t] eq $r)) {
  2712. if($c < $max) {
  2713. $log_line .= ", " if($c);
  2714. $log_line .= $t;
  2715. }
  2716. $c++;
  2717. }
  2718. }
  2719. if($c > $max) {
  2720. $log_line .= " and ".($c-$max)." more";
  2721. }
  2722. $log_line .= ")\n";
  2723. $restraints{$log_line} = $skip_count;
  2724. }
  2725. foreach my $log_line (sort {$restraints{$b} <=> $restraints{$a}} keys %restraints) {
  2726. logmsg $log_line;
  2727. }
  2728. }
  2729. if($total) {
  2730. if($failedign) {
  2731. my $failedignsorted = numsortwords($failedign);
  2732. logmsg "IGNORED: failed tests: $failedignsorted\n";
  2733. }
  2734. logmsg sprintf("TESTDONE: $ok tests out of $total reported OK: %d%%\n",
  2735. $ok/$total*100);
  2736. if($failed && ($ok != $total)) {
  2737. my $failedsorted = numsortwords($failed);
  2738. logmsg "\nTESTFAIL: These test cases failed: $failedsorted\n\n";
  2739. }
  2740. }
  2741. else {
  2742. logmsg "\nTESTFAIL: No tests were performed\n\n";
  2743. if(scalar(keys %enabled_keywords)) {
  2744. logmsg "TESTFAIL: Nothing matched these keywords: ";
  2745. for(keys %enabled_keywords) {
  2746. logmsg "$_ ";
  2747. }
  2748. logmsg "\n";
  2749. }
  2750. }
  2751. if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
  2752. exit 1;
  2753. }