Test.pm 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327
  1. # Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License 2.0 (the "License"). You may not use
  4. # this file except in compliance with the License. You can obtain a copy
  5. # in the file LICENSE in the source distribution or at
  6. # https://www.openssl.org/source/license.html
  7. package OpenSSL::Test;
  8. use strict;
  9. use warnings;
  10. use Carp;
  11. use Test::More 0.96;
  12. use Exporter;
  13. use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
  14. $VERSION = "1.0";
  15. @ISA = qw(Exporter);
  16. @EXPORT = (@Test::More::EXPORT, qw(setup run indir cmd app fuzz test
  17. perlapp perltest subtest));
  18. @EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file
  19. srctop_dir srctop_file
  20. data_file data_dir
  21. result_file result_dir
  22. pipe with cmdstr
  23. openssl_versions
  24. ok_nofips is_nofips isnt_nofips));
  25. =head1 NAME
  26. OpenSSL::Test - a private extension of Test::More
  27. =head1 SYNOPSIS
  28. use OpenSSL::Test;
  29. setup("my_test_name");
  30. plan tests => 2;
  31. ok(run(app(["openssl", "version"])), "check for openssl presence");
  32. indir "subdir" => sub {
  33. ok(run(test(["sometest", "arg1"], stdout => "foo.txt")),
  34. "run sometest with output to foo.txt");
  35. };
  36. =head1 DESCRIPTION
  37. This module is a private extension of L<Test::More> for testing OpenSSL.
  38. In addition to the Test::More functions, it also provides functions that
  39. easily find the diverse programs within a OpenSSL build tree, as well as
  40. some other useful functions.
  41. This module I<depends> on the environment variables C<$TOP> or C<$SRCTOP>
  42. and C<$BLDTOP>. Without one of the combinations it refuses to work.
  43. See L</ENVIRONMENT> below.
  44. With each test recipe, a parallel data directory with (almost) the same name
  45. as the recipe is possible in the source directory tree. For example, for a
  46. recipe C<$SRCTOP/test/recipes/99-foo.t>, there could be a directory
  47. C<$SRCTOP/test/recipes/99-foo_data/>.
  48. =cut
  49. use File::Copy;
  50. use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir
  51. catdir catfile splitpath catpath devnull abs2rel/;
  52. use File::Path 2.00 qw/rmtree mkpath/;
  53. use File::Basename;
  54. use Cwd qw/getcwd abs_path/;
  55. use OpenSSL::Util;
  56. my $level = 0;
  57. # The name of the test. This is set by setup() and is used in the other
  58. # functions to verify that setup() has been used.
  59. my $test_name = undef;
  60. # Directories we want to keep track of TOP, APPS, TEST and RESULTS are the
  61. # ones we're interested in, corresponding to the environment variables TOP
  62. # (mandatory), BIN_D, TEST_D, UTIL_D and RESULT_D.
  63. my %directories = ();
  64. # The environment variables that gave us the contents in %directories. These
  65. # get modified whenever we change directories, so that subprocesses can use
  66. # the values of those environment variables as well
  67. my @direnv = ();
  68. # A bool saying if we shall stop all testing if the current recipe has failing
  69. # tests or not. This is set by setup() if the environment variable STOPTEST
  70. # is defined with a non-empty value.
  71. my $end_with_bailout = 0;
  72. # A set of hooks that is affected by with() and may be used in diverse places.
  73. # All hooks are expected to be CODE references.
  74. my %hooks = (
  75. # exit_checker is used by run() directly after completion of a command.
  76. # it receives the exit code from that command and is expected to return
  77. # 1 (for success) or 0 (for failure). This is the status value that run()
  78. # will give back (through the |statusvar| reference and as returned value
  79. # when capture => 1 doesn't apply).
  80. exit_checker => sub { return shift == 0 ? 1 : 0 },
  81. );
  82. # Debug flag, to be set manually when needed
  83. my $debug = 0;
  84. =head2 Main functions
  85. The following functions are exported by default when using C<OpenSSL::Test>.
  86. =cut
  87. =over 4
  88. =item B<setup "NAME">
  89. C<setup> is used for initial setup, and it is mandatory that it's used.
  90. If it's not used in a OpenSSL test recipe, the rest of the recipe will
  91. most likely refuse to run.
  92. C<setup> checks for environment variables (see L</ENVIRONMENT> below),
  93. checks that C<$TOP/Configure> or C<$SRCTOP/Configure> exists, C<chdir>
  94. into the results directory (defined by the C<$RESULT_D> environment
  95. variable if defined, otherwise C<$BLDTOP/test-runs> or C<$TOP/test-runs>,
  96. whichever is defined).
  97. =back
  98. =cut
  99. sub setup {
  100. my $old_test_name = $test_name;
  101. $test_name = shift;
  102. my %opts = @_;
  103. BAIL_OUT("setup() must receive a name") unless $test_name;
  104. warn "setup() detected test name change. Innocuous, so we continue...\n"
  105. if $old_test_name && $old_test_name ne $test_name;
  106. return if $old_test_name;
  107. BAIL_OUT("setup() needs \$TOP or \$SRCTOP and \$BLDTOP to be defined")
  108. unless $ENV{TOP} || ($ENV{SRCTOP} && $ENV{BLDTOP});
  109. BAIL_OUT("setup() found both \$TOP and \$SRCTOP or \$BLDTOP...")
  110. if $ENV{TOP} && ($ENV{SRCTOP} || $ENV{BLDTOP});
  111. __env();
  112. BAIL_OUT("setup() expects the file Configure in the source top directory")
  113. unless -f srctop_file("Configure");
  114. note "The results of this test will end up in $directories{RESULTS}"
  115. unless $opts{quiet};
  116. __cwd($directories{RESULTS});
  117. }
  118. =over 4
  119. =item B<indir "SUBDIR" =E<gt> sub BLOCK, OPTS>
  120. C<indir> is used to run a part of the recipe in a different directory than
  121. the one C<setup> moved into, usually a subdirectory, given by SUBDIR.
  122. The part of the recipe that's run there is given by the codeblock BLOCK.
  123. C<indir> takes some additional options OPTS that affect the subdirectory:
  124. =over 4
  125. =item B<create =E<gt> 0|1>
  126. When set to 1 (or any value that perl perceives as true), the subdirectory
  127. will be created if it doesn't already exist. This happens before BLOCK
  128. is executed.
  129. =back
  130. An example:
  131. indir "foo" => sub {
  132. ok(run(app(["openssl", "version"]), stdout => "foo.txt"));
  133. if (ok(open(RESULT, "foo.txt"), "reading foo.txt")) {
  134. my $line = <RESULT>;
  135. close RESULT;
  136. is($line, qr/^OpenSSL 1\./,
  137. "check that we're using OpenSSL 1.x.x");
  138. }
  139. }, create => 1;
  140. =back
  141. =cut
  142. sub indir {
  143. my $subdir = shift;
  144. my $codeblock = shift;
  145. my %opts = @_;
  146. my $reverse = __cwd($subdir,%opts);
  147. BAIL_OUT("FAILURE: indir, \"$subdir\" wasn't possible to move into")
  148. unless $reverse;
  149. $codeblock->();
  150. __cwd($reverse);
  151. }
  152. =over 4
  153. =item B<cmd ARRAYREF, OPTS>
  154. This functions build up a platform dependent command based on the
  155. input. It takes a reference to a list that is the executable or
  156. script and its arguments, and some additional options (described
  157. further on). Where necessary, the command will be wrapped in a
  158. suitable environment to make sure the correct shared libraries are
  159. used (currently only on Unix).
  160. It returns a CODEREF to be used by C<run>, C<pipe> or C<cmdstr>.
  161. The options that C<cmd> (as well as its derivatives described below) can take
  162. are in the form of hash values:
  163. =over 4
  164. =item B<stdin =E<gt> PATH>
  165. =item B<stdout =E<gt> PATH>
  166. =item B<stderr =E<gt> PATH>
  167. In all three cases, the corresponding standard input, output or error is
  168. redirected from (for stdin) or to (for the others) a file given by the
  169. string PATH, I<or>, if the value is C<undef>, C</dev/null> or similar.
  170. =back
  171. =item B<app ARRAYREF, OPTS>
  172. =item B<test ARRAYREF, OPTS>
  173. Both of these are specific applications of C<cmd>, with just a couple
  174. of small difference:
  175. C<app> expects to find the given command (the first item in the given list
  176. reference) as an executable in C<$BIN_D> (if defined, otherwise C<$TOP/apps>
  177. or C<$BLDTOP/apps>).
  178. C<test> expects to find the given command (the first item in the given list
  179. reference) as an executable in C<$TEST_D> (if defined, otherwise C<$TOP/test>
  180. or C<$BLDTOP/test>).
  181. Also, for both C<app> and C<test>, the command may be prefixed with
  182. the content of the environment variable C<$EXE_SHELL>, which is useful
  183. in case OpenSSL has been cross compiled.
  184. =item B<perlapp ARRAYREF, OPTS>
  185. =item B<perltest ARRAYREF, OPTS>
  186. These are also specific applications of C<cmd>, where the interpreter
  187. is predefined to be C<perl>, and they expect the script to be
  188. interpreted to reside in the same location as C<app> and C<test>.
  189. C<perlapp> and C<perltest> will also take the following option:
  190. =over 4
  191. =item B<interpreter_args =E<gt> ARRAYref>
  192. The array reference is a set of arguments for the interpreter rather
  193. than the script. Take care so that none of them can be seen as a
  194. script! Flags and their eventual arguments only!
  195. =back
  196. An example:
  197. ok(run(perlapp(["foo.pl", "arg1"],
  198. interpreter_args => [ "-I", srctop_dir("test") ])));
  199. =back
  200. =begin comment
  201. One might wonder over the complexity of C<apps>, C<fuzz>, C<test>, ...
  202. with all the lazy evaluations and all that. The reason for this is that
  203. we want to make sure the directory in which those programs are found are
  204. correct at the time these commands are used. Consider the following code
  205. snippet:
  206. my $cmd = app(["openssl", ...]);
  207. indir "foo", sub {
  208. ok(run($cmd), "Testing foo")
  209. };
  210. If there wasn't this lazy evaluation, the directory where C<openssl> is
  211. found would be incorrect at the time C<run> is called, because it was
  212. calculated before we moved into the directory "foo".
  213. =end comment
  214. =cut
  215. sub cmd {
  216. my $cmd = shift;
  217. my %opts = @_;
  218. return sub {
  219. my $num = shift;
  220. # Make a copy to not destroy the caller's array
  221. my @cmdargs = ( @$cmd );
  222. my @prog = __wrap_cmd(shift @cmdargs, $opts{exe_shell} // ());
  223. return __decorate_cmd($num, [ @prog, fixup_cmd_elements(@cmdargs) ],
  224. %opts);
  225. }
  226. }
  227. sub app {
  228. my $cmd = shift;
  229. my %opts = @_;
  230. return sub {
  231. my @cmdargs = ( @{$cmd} );
  232. my @prog = __fixup_prg(__apps_file(shift @cmdargs, __exeext()));
  233. return cmd([ @prog, @cmdargs ],
  234. exe_shell => $ENV{EXE_SHELL}, %opts) -> (shift);
  235. }
  236. }
  237. sub fuzz {
  238. my $cmd = shift;
  239. my %opts = @_;
  240. return sub {
  241. my @cmdargs = ( @{$cmd} );
  242. my @prog = __fixup_prg(__fuzz_file(shift @cmdargs, __exeext()));
  243. return cmd([ @prog, @cmdargs ],
  244. exe_shell => $ENV{EXE_SHELL}, %opts) -> (shift);
  245. }
  246. }
  247. sub test {
  248. my $cmd = shift;
  249. my %opts = @_;
  250. return sub {
  251. my @cmdargs = ( @{$cmd} );
  252. my @prog = __fixup_prg(__test_file(shift @cmdargs, __exeext()));
  253. return cmd([ @prog, @cmdargs ],
  254. exe_shell => $ENV{EXE_SHELL}, %opts) -> (shift);
  255. }
  256. }
  257. sub perlapp {
  258. my $cmd = shift;
  259. my %opts = @_;
  260. return sub {
  261. my @interpreter_args = defined $opts{interpreter_args} ?
  262. @{$opts{interpreter_args}} : ();
  263. my @interpreter = __fixup_prg($^X);
  264. my @cmdargs = ( @{$cmd} );
  265. my @prog = __apps_file(shift @cmdargs, undef);
  266. return cmd([ @interpreter, @interpreter_args,
  267. @prog, @cmdargs ], %opts) -> (shift);
  268. }
  269. }
  270. sub perltest {
  271. my $cmd = shift;
  272. my %opts = @_;
  273. return sub {
  274. my @interpreter_args = defined $opts{interpreter_args} ?
  275. @{$opts{interpreter_args}} : ();
  276. my @interpreter = __fixup_prg($^X);
  277. my @cmdargs = ( @{$cmd} );
  278. my @prog = __test_file(shift @cmdargs, undef);
  279. return cmd([ @interpreter, @interpreter_args,
  280. @prog, @cmdargs ], %opts) -> (shift);
  281. }
  282. }
  283. =over 4
  284. =item B<run CODEREF, OPTS>
  285. CODEREF is expected to be the value return by C<cmd> or any of its
  286. derivatives, anything else will most likely cause an error unless you
  287. know what you're doing.
  288. C<run> executes the command returned by CODEREF and return either the
  289. resulting standard output (if the option C<capture> is set true) or a boolean
  290. indicating if the command succeeded or not.
  291. The options that C<run> can take are in the form of hash values:
  292. =over 4
  293. =item B<capture =E<gt> 0|1>
  294. If true, the command will be executed with a perl backtick,
  295. and C<run> will return the resulting standard output as an array of lines.
  296. If false or not given, the command will be executed with C<system()>,
  297. and C<run> will return 1 if the command was successful or 0 if it wasn't.
  298. =item B<prefix =E<gt> EXPR>
  299. If specified, EXPR will be used as a string to prefix the output from the
  300. command. This is useful if the output contains lines starting with C<ok >
  301. or C<not ok > that can disturb Test::Harness.
  302. =item B<statusvar =E<gt> VARREF>
  303. If used, B<VARREF> must be a reference to a scalar variable. It will be
  304. assigned a boolean indicating if the command succeeded or not. This is
  305. particularly useful together with B<capture>.
  306. =back
  307. Usually 1 indicates that the command was successful and 0 indicates failure.
  308. For further discussion on what is considered a successful command or not, see
  309. the function C<with> further down.
  310. =back
  311. =cut
  312. sub run {
  313. my ($cmd, $display_cmd) = shift->(0);
  314. my %opts = @_;
  315. return () if !$cmd;
  316. my $prefix = "";
  317. if ( $^O eq "VMS" ) { # VMS
  318. $prefix = "pipe ";
  319. }
  320. my @r = ();
  321. my $r = 0;
  322. my $e = 0;
  323. die "OpenSSL::Test::run(): statusvar value not a scalar reference"
  324. if $opts{statusvar} && ref($opts{statusvar}) ne "SCALAR";
  325. # For some reason, program output, or even output from this function
  326. # somehow isn't caught by TAP::Harness (TAP::Parser?) on VMS, so we're
  327. # silencing it specifically there until further notice.
  328. my $save_STDOUT;
  329. my $save_STDERR;
  330. if ($^O eq 'VMS') {
  331. # In non-verbose, we want to shut up the command interpreter, in case
  332. # it has something to complain about. On VMS, it might complain both
  333. # on stdout and stderr
  334. if ($ENV{HARNESS_ACTIVE} && !$ENV{HARNESS_VERBOSE}) {
  335. open $save_STDOUT, '>&', \*STDOUT or die "Can't dup STDOUT: $!";
  336. open $save_STDERR, '>&', \*STDERR or die "Can't dup STDERR: $!";
  337. open STDOUT, ">", devnull();
  338. open STDERR, ">", devnull();
  339. }
  340. }
  341. $ENV{HARNESS_OSSL_LEVEL} = $level + 1;
  342. # The dance we do with $? is the same dance the Unix shells appear to
  343. # do. For example, a program that gets aborted (and therefore signals
  344. # SIGABRT = 6) will appear to exit with the code 134. We mimic this
  345. # to make it easier to compare with a manual run of the command.
  346. if ($opts{capture} || defined($opts{prefix})) {
  347. my $pipe;
  348. local $_;
  349. open($pipe, '-|', "$prefix$cmd") or die "Can't start command: $!";
  350. while(<$pipe>) {
  351. my $l = ($opts{prefix} // "") . $_;
  352. if ($opts{capture}) {
  353. push @r, $l;
  354. } else {
  355. print STDOUT $l;
  356. }
  357. }
  358. close $pipe;
  359. } else {
  360. $ENV{HARNESS_OSSL_PREFIX} = "# ";
  361. system("$prefix$cmd");
  362. delete $ENV{HARNESS_OSSL_PREFIX};
  363. }
  364. $e = ($? & 0x7f) ? ($? & 0x7f)|0x80 : ($? >> 8);
  365. $r = $hooks{exit_checker}->($e);
  366. if ($opts{statusvar}) {
  367. ${$opts{statusvar}} = $r;
  368. }
  369. # Restore STDOUT / STDERR on VMS
  370. if ($^O eq 'VMS') {
  371. if ($ENV{HARNESS_ACTIVE} && !$ENV{HARNESS_VERBOSE}) {
  372. close STDOUT;
  373. close STDERR;
  374. open STDOUT, '>&', $save_STDOUT or die "Can't restore STDOUT: $!";
  375. open STDERR, '>&', $save_STDERR or die "Can't restore STDERR: $!";
  376. }
  377. print STDERR "$prefix$display_cmd => $e\n"
  378. if !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
  379. } else {
  380. print STDERR "$prefix$display_cmd => $e\n";
  381. }
  382. # At this point, $? stops being interesting, and unfortunately,
  383. # there are Test::More versions that get picky if we leave it
  384. # non-zero.
  385. $? = 0;
  386. if ($opts{capture}) {
  387. return @r;
  388. } else {
  389. return $r;
  390. }
  391. }
  392. END {
  393. my $tb = Test::More->builder;
  394. my $failure = scalar(grep { $_ == 0; } $tb->summary);
  395. if ($failure && $end_with_bailout) {
  396. BAIL_OUT("Stoptest!");
  397. }
  398. }
  399. =head2 Utility functions
  400. The following functions are exported on request when using C<OpenSSL::Test>.
  401. # To only get the bldtop_file and srctop_file functions.
  402. use OpenSSL::Test qw/bldtop_file srctop_file/;
  403. # To only get the bldtop_file function in addition to the default ones.
  404. use OpenSSL::Test qw/:DEFAULT bldtop_file/;
  405. =cut
  406. # Utility functions, exported on request
  407. =over 4
  408. =item B<bldtop_dir LIST>
  409. LIST is a list of directories that make up a path from the top of the OpenSSL
  410. build directory (as indicated by the environment variable C<$TOP> or
  411. C<$BLDTOP>).
  412. C<bldtop_dir> returns the resulting directory as a string, adapted to the local
  413. operating system.
  414. =back
  415. =cut
  416. sub bldtop_dir {
  417. my $d = __bldtop_dir(@_); # This caters for operating systems that have
  418. # a very distinct syntax for directories.
  419. croak "$d isn't a directory" if -e $d && ! -d $d;
  420. return $d;
  421. }
  422. =over 4
  423. =item B<bldtop_file LIST, FILENAME>
  424. LIST is a list of directories that make up a path from the top of the OpenSSL
  425. build directory (as indicated by the environment variable C<$TOP> or
  426. C<$BLDTOP>) and FILENAME is the name of a file located in that directory path.
  427. C<bldtop_file> returns the resulting file path as a string, adapted to the local
  428. operating system.
  429. =back
  430. =cut
  431. sub bldtop_file {
  432. my $f = __bldtop_file(@_);
  433. croak "$f isn't a file" if -e $f && ! -f $f;
  434. return $f;
  435. }
  436. =over 4
  437. =item B<srctop_dir LIST>
  438. LIST is a list of directories that make up a path from the top of the OpenSSL
  439. source directory (as indicated by the environment variable C<$TOP> or
  440. C<$SRCTOP>).
  441. C<srctop_dir> returns the resulting directory as a string, adapted to the local
  442. operating system.
  443. =back
  444. =cut
  445. sub srctop_dir {
  446. my $d = __srctop_dir(@_); # This caters for operating systems that have
  447. # a very distinct syntax for directories.
  448. croak "$d isn't a directory" if -e $d && ! -d $d;
  449. return $d;
  450. }
  451. =over 4
  452. =item B<srctop_file LIST, FILENAME>
  453. LIST is a list of directories that make up a path from the top of the OpenSSL
  454. source directory (as indicated by the environment variable C<$TOP> or
  455. C<$SRCTOP>) and FILENAME is the name of a file located in that directory path.
  456. C<srctop_file> returns the resulting file path as a string, adapted to the local
  457. operating system.
  458. =back
  459. =cut
  460. sub srctop_file {
  461. my $f = __srctop_file(@_);
  462. croak "$f isn't a file" if -e $f && ! -f $f;
  463. return $f;
  464. }
  465. =over 4
  466. =item B<data_dir LIST>
  467. LIST is a list of directories that make up a path from the data directory
  468. associated with the test (see L</DESCRIPTION> above).
  469. C<data_dir> returns the resulting directory as a string, adapted to the local
  470. operating system.
  471. =back
  472. =cut
  473. sub data_dir {
  474. my $d = __data_dir(@_);
  475. croak "$d isn't a directory" if -e $d && ! -d $d;
  476. return $d;
  477. }
  478. =over 4
  479. =item B<data_file LIST, FILENAME>
  480. LIST is a list of directories that make up a path from the data directory
  481. associated with the test (see L</DESCRIPTION> above) and FILENAME is the name
  482. of a file located in that directory path. C<data_file> returns the resulting
  483. file path as a string, adapted to the local operating system.
  484. =back
  485. =cut
  486. sub data_file {
  487. my $f = __data_file(@_);
  488. croak "$f isn't a file" if -e $f && ! -f $f;
  489. return $f;
  490. }
  491. =over 4
  492. =item B<result_dir LIST>
  493. LIST is a list of directories that make up a path from the result directory
  494. associated with the test (see L</DESCRIPTION> above).
  495. C<result_dir> returns the resulting directory as a string, adapted to the local
  496. operating system.
  497. =back
  498. =cut
  499. sub result_dir {
  500. BAIL_OUT("Must run setup() first") if (! $test_name);
  501. my $d = catdir($directories{RESULTS},@_);
  502. croak "$d isn't a directory" if -e $d && ! -d $d;
  503. return $d;
  504. }
  505. =over 4
  506. =item B<result_file LIST, FILENAME>
  507. LIST is a list of directories that make up a path from the data directory
  508. associated with the test (see L</DESCRIPTION> above) and FILENAME is the name
  509. of a file located in that directory path. C<result_file> returns the resulting
  510. file path as a string, adapted to the local operating system.
  511. =back
  512. =cut
  513. sub result_file {
  514. BAIL_OUT("Must run setup() first") if (! $test_name);
  515. my $f = catfile(result_dir(),@_);
  516. croak "$f isn't a file" if -e $f && ! -f $f;
  517. return $f;
  518. }
  519. =over 4
  520. =item B<pipe LIST>
  521. LIST is a list of CODEREFs returned by C<app> or C<test>, from which C<pipe>
  522. creates a new command composed of all the given commands put together in a
  523. pipe. C<pipe> returns a new CODEREF in the same manner as C<app> or C<test>,
  524. to be passed to C<run> for execution.
  525. =back
  526. =cut
  527. sub pipe {
  528. my @cmds = @_;
  529. return
  530. sub {
  531. my @cs = ();
  532. my @dcs = ();
  533. my @els = ();
  534. my $counter = 0;
  535. foreach (@cmds) {
  536. my ($c, $dc, @el) = $_->(++$counter);
  537. return () if !$c;
  538. push @cs, $c;
  539. push @dcs, $dc;
  540. push @els, @el;
  541. }
  542. return (
  543. join(" | ", @cs),
  544. join(" | ", @dcs),
  545. @els
  546. );
  547. };
  548. }
  549. =over 4
  550. =item B<with HASHREF, CODEREF>
  551. C<with> will temporarily install hooks given by the HASHREF and then execute
  552. the given CODEREF. Hooks are usually expected to have a coderef as value.
  553. The currently available hoosk are:
  554. =over 4
  555. =item B<exit_checker =E<gt> CODEREF>
  556. This hook is executed after C<run> has performed its given command. The
  557. CODEREF receives the exit code as only argument and is expected to return
  558. 1 (if the exit code indicated success) or 0 (if the exit code indicated
  559. failure).
  560. =back
  561. =back
  562. =cut
  563. sub with {
  564. my $opts = shift;
  565. my %opts = %{$opts};
  566. my $codeblock = shift;
  567. my %saved_hooks = ();
  568. foreach (keys %opts) {
  569. $saved_hooks{$_} = $hooks{$_} if exists($hooks{$_});
  570. $hooks{$_} = $opts{$_};
  571. }
  572. $codeblock->();
  573. foreach (keys %saved_hooks) {
  574. $hooks{$_} = $saved_hooks{$_};
  575. }
  576. }
  577. =over 4
  578. =item B<cmdstr CODEREF, OPTS>
  579. C<cmdstr> takes a CODEREF from C<app> or C<test> and simply returns the
  580. command as a string.
  581. C<cmdstr> takes some additional options OPTS that affect the string returned:
  582. =over 4
  583. =item B<display =E<gt> 0|1>
  584. When set to 0, the returned string will be with all decorations, such as a
  585. possible redirect of stderr to the null device. This is suitable if the
  586. string is to be used directly in a recipe.
  587. When set to 1, the returned string will be without extra decorations. This
  588. is suitable for display if that is desired (doesn't confuse people with all
  589. internal stuff), or if it's used to pass a command down to a subprocess.
  590. Default: 0
  591. =back
  592. =back
  593. =cut
  594. sub cmdstr {
  595. my ($cmd, $display_cmd) = shift->(0);
  596. my %opts = @_;
  597. if ($opts{display}) {
  598. return $display_cmd;
  599. } else {
  600. return $cmd;
  601. }
  602. }
  603. =over 4
  604. =over 4
  605. =item B<openssl_versions>
  606. Returns a list of two version numbers, the first representing the build
  607. version, the second representing the library version. See opensslv.h for
  608. more information on those numbers.
  609. =back
  610. =cut
  611. my @versions = ();
  612. sub openssl_versions {
  613. unless (@versions) {
  614. my %lines =
  615. map { s/\R$//;
  616. /^(.*): (.*)$/;
  617. $1 => $2 }
  618. run(test(['versions']), capture => 1);
  619. @versions = ( $lines{'Build version'}, $lines{'Library version'} );
  620. }
  621. return @versions;
  622. }
  623. =over 4
  624. =item B<ok_nofips EXPR, TEST_NAME>
  625. C<ok_nofips> is equivalent to using C<ok> when the environment variable
  626. C<FIPS_MODE> is undefined, otherwise it is equivalent to C<not ok>. This can be
  627. used for C<ok> tests that must fail when testing a FIPS provider. The parameters
  628. are the same as used by C<ok> which is an expression EXPR followed by the test
  629. description TEST_NAME.
  630. An example:
  631. ok_nofips(run(app(["md5.pl"])), "md5 should fail in fips mode");
  632. =item B<is_nofips EXPR1, EXPR2, TEST_NAME>
  633. C<is_nofips> is equivalent to using C<is> when the environment variable
  634. C<FIPS_MODE> is undefined, otherwise it is equivalent to C<isnt>. This can be
  635. used for C<is> tests that must fail when testing a FIPS provider. The parameters
  636. are the same as used by C<is> which has 2 arguments EXPR1 and EXPR2 that can be
  637. compared using eq or ne, followed by a test description TEST_NAME.
  638. An example:
  639. is_nofips(ultimate_answer(), 42, "Meaning of Life");
  640. =item B<isnt_nofips EXPR1, EXPR2, TEST_NAME>
  641. C<isnt_nofips> is equivalent to using C<isnt> when the environment variable
  642. C<FIPS_MODE> is undefined, otherwise it is equivalent to C<is>. This can be
  643. used for C<isnt> tests that must fail when testing a FIPS provider. The
  644. parameters are the same as used by C<isnt> which has 2 arguments EXPR1 and EXPR2
  645. that can be compared using ne or eq, followed by a test description TEST_NAME.
  646. An example:
  647. isnt_nofips($foo, '', "Got some foo");
  648. =back
  649. =cut
  650. sub ok_nofips {
  651. return ok(!$_[0], @_[1..$#_]) if defined $ENV{FIPS_MODE};
  652. return ok($_[0], @_[1..$#_]);
  653. }
  654. sub is_nofips {
  655. return isnt($_[0], $_[1], @_[2..$#_]) if defined $ENV{FIPS_MODE};
  656. return is($_[0], $_[1], @_[2..$#_]);
  657. }
  658. sub isnt_nofips {
  659. return is($_[0], $_[1], @_[2..$#_]) if defined $ENV{FIPS_MODE};
  660. return isnt($_[0], $_[1], @_[2..$#_]);
  661. }
  662. ######################################################################
  663. # private functions. These are never exported.
  664. =head1 ENVIRONMENT
  665. OpenSSL::Test depends on some environment variables.
  666. =over 4
  667. =item B<TOP>
  668. This environment variable is mandatory. C<setup> will check that it's
  669. defined and that it's a directory that contains the file C<Configure>.
  670. If this isn't so, C<setup> will C<BAIL_OUT>.
  671. =item B<BIN_D>
  672. If defined, its value should be the directory where the openssl application
  673. is located. Defaults to C<$TOP/apps> (adapted to the operating system).
  674. =item B<TEST_D>
  675. If defined, its value should be the directory where the test applications
  676. are located. Defaults to C<$TOP/test> (adapted to the operating system).
  677. =item B<STOPTEST>
  678. If defined, it puts testing in a different mode, where a recipe with
  679. failures will result in a C<BAIL_OUT> at the end of its run.
  680. =item B<FIPS_MODE>
  681. If defined it indicates that the FIPS provider is being tested. Tests may use
  682. B<ok_nofips>, B<is_nofips> and B<isnt_nofips> to invert test results
  683. i.e. Some tests may only work in non FIPS mode.
  684. =back
  685. =cut
  686. sub __env {
  687. (my $recipe_datadir = basename($0)) =~ s/\.t$/_data/i;
  688. $directories{SRCTOP} = abs_path($ENV{SRCTOP} || $ENV{TOP});
  689. $directories{BLDTOP} = abs_path($ENV{BLDTOP} || $ENV{TOP});
  690. $directories{BLDAPPS} = $ENV{BIN_D} || __bldtop_dir("apps");
  691. $directories{SRCAPPS} = __srctop_dir("apps");
  692. $directories{BLDFUZZ} = __bldtop_dir("fuzz");
  693. $directories{SRCFUZZ} = __srctop_dir("fuzz");
  694. $directories{BLDTEST} = $ENV{TEST_D} || __bldtop_dir("test");
  695. $directories{SRCTEST} = __srctop_dir("test");
  696. $directories{SRCDATA} = __srctop_dir("test", "recipes",
  697. $recipe_datadir);
  698. $directories{RESULTTOP} = $ENV{RESULT_D} || __bldtop_dir("test-runs");
  699. $directories{RESULTS} = catdir($directories{RESULTTOP}, $test_name);
  700. # Create result directory dynamically
  701. rmtree($directories{RESULTS}, { safe => 0, keep_root => 1 });
  702. mkpath($directories{RESULTS});
  703. # All directories are assumed to exist, except for SRCDATA. If that one
  704. # doesn't exist, just drop it.
  705. delete $directories{SRCDATA} unless -d $directories{SRCDATA};
  706. push @direnv, "TOP" if $ENV{TOP};
  707. push @direnv, "SRCTOP" if $ENV{SRCTOP};
  708. push @direnv, "BLDTOP" if $ENV{BLDTOP};
  709. push @direnv, "BIN_D" if $ENV{BIN_D};
  710. push @direnv, "TEST_D" if $ENV{TEST_D};
  711. push @direnv, "RESULT_D" if $ENV{RESULT_D};
  712. $end_with_bailout = $ENV{STOPTEST} ? 1 : 0;
  713. };
  714. # __srctop_file and __srctop_dir are helpers to build file and directory
  715. # names on top of the source directory. They depend on $SRCTOP, and
  716. # therefore on the proper use of setup() and when needed, indir().
  717. # __bldtop_file and __bldtop_dir do the same thing but relative to $BLDTOP.
  718. # __srctop_file and __bldtop_file take the same kind of argument as
  719. # File::Spec::Functions::catfile.
  720. # Similarly, __srctop_dir and __bldtop_dir take the same kind of argument
  721. # as File::Spec::Functions::catdir
  722. sub __srctop_file {
  723. BAIL_OUT("Must run setup() first") if (! $test_name);
  724. my $f = pop;
  725. return abs2rel(catfile($directories{SRCTOP},@_,$f),getcwd);
  726. }
  727. sub __srctop_dir {
  728. BAIL_OUT("Must run setup() first") if (! $test_name);
  729. return abs2rel(catdir($directories{SRCTOP},@_), getcwd);
  730. }
  731. sub __bldtop_file {
  732. BAIL_OUT("Must run setup() first") if (! $test_name);
  733. my $f = pop;
  734. return abs2rel(catfile($directories{BLDTOP},@_,$f), getcwd);
  735. }
  736. sub __bldtop_dir {
  737. BAIL_OUT("Must run setup() first") if (! $test_name);
  738. return abs2rel(catdir($directories{BLDTOP},@_), getcwd);
  739. }
  740. # __exeext is a function that returns the platform dependent file extension
  741. # for executable binaries, or the value of the environment variable $EXE_EXT
  742. # if that one is defined.
  743. sub __exeext {
  744. my $ext = "";
  745. if ($^O eq "VMS" ) { # VMS
  746. $ext = ".exe";
  747. } elsif ($^O eq "MSWin32") { # Windows
  748. $ext = ".exe";
  749. }
  750. return $ENV{"EXE_EXT"} || $ext;
  751. }
  752. # __test_file, __apps_file and __fuzz_file return the full path to a file
  753. # relative to the test/, apps/ or fuzz/ directory in the build tree or the
  754. # source tree, depending on where the file is found. Note that when looking
  755. # in the build tree, the file name with an added extension is looked for, if
  756. # an extension is given. The intent is to look for executable binaries (in
  757. # the build tree) or possibly scripts (in the source tree).
  758. # These functions all take the same arguments as File::Spec::Functions::catfile,
  759. # *plus* a mandatory extension argument. This extension argument can be undef,
  760. # and is ignored in such a case.
  761. sub __test_file {
  762. BAIL_OUT("Must run setup() first") if (! $test_name);
  763. my $e = pop || "";
  764. my $f = pop;
  765. my $out = catfile($directories{BLDTEST},@_,$f . $e);
  766. $out = catfile($directories{SRCTEST},@_,$f) unless -f $out;
  767. return $out;
  768. }
  769. sub __apps_file {
  770. BAIL_OUT("Must run setup() first") if (! $test_name);
  771. my $e = pop || "";
  772. my $f = pop;
  773. my $out = catfile($directories{BLDAPPS},@_,$f . $e);
  774. $out = catfile($directories{SRCAPPS},@_,$f) unless -f $out;
  775. return $out;
  776. }
  777. sub __fuzz_file {
  778. BAIL_OUT("Must run setup() first") if (! $test_name);
  779. my $e = pop || "";
  780. my $f = pop;
  781. my $out = catfile($directories{BLDFUZZ},@_,$f . $e);
  782. $out = catfile($directories{SRCFUZZ},@_,$f) unless -f $out;
  783. return $out;
  784. }
  785. sub __data_file {
  786. BAIL_OUT("Must run setup() first") if (! $test_name);
  787. return undef unless exists $directories{SRCDATA};
  788. my $f = pop;
  789. return catfile($directories{SRCDATA},@_,$f);
  790. }
  791. sub __data_dir {
  792. BAIL_OUT("Must run setup() first") if (! $test_name);
  793. return undef unless exists $directories{SRCDATA};
  794. return catdir($directories{SRCDATA},@_);
  795. }
  796. # __cwd DIR
  797. # __cwd DIR, OPTS
  798. #
  799. # __cwd changes directory to DIR (string) and changes all the relative
  800. # entries in %directories accordingly. OPTS is an optional series of
  801. # hash style arguments to alter __cwd's behavior:
  802. #
  803. # create = 0|1 The directory we move to is created if 1, not if 0.
  804. sub __cwd {
  805. my $dir = catdir(shift);
  806. my %opts = @_;
  807. # If the directory is to be created, we must do that before using
  808. # abs_path().
  809. $dir = canonpath($dir);
  810. if ($opts{create}) {
  811. mkpath($dir);
  812. }
  813. my $abscurdir = abs_path(curdir());
  814. my $absdir = abs_path($dir);
  815. my $reverse = abs2rel($abscurdir, $absdir);
  816. # PARANOIA: if we're not moving anywhere, we do nothing more
  817. if ($abscurdir eq $absdir) {
  818. return $reverse;
  819. }
  820. # Do not support a move to a different volume for now. Maybe later.
  821. BAIL_OUT("FAILURE: \"$dir\" moves to a different volume, not supported")
  822. if $reverse eq $abscurdir;
  823. # If someone happened to give a directory that leads back to the current,
  824. # it's extremely silly to do anything more, so just simulate that we did
  825. # move.
  826. # In this case, we won't even clean it out, for safety's sake.
  827. return "." if $reverse eq "";
  828. # We are recalculating the directories we keep track of, but need to save
  829. # away the result for after having moved into the new directory.
  830. my %tmp_directories = ();
  831. my %tmp_ENV = ();
  832. # For each of these directory variables, figure out where they are relative
  833. # to the directory we want to move to if they aren't absolute (if they are,
  834. # they don't change!)
  835. my @dirtags = sort keys %directories;
  836. foreach (@dirtags) {
  837. if (!file_name_is_absolute($directories{$_})) {
  838. my $oldpath = abs_path($directories{$_});
  839. my $newpath = abs2rel($oldpath, $absdir);
  840. if ($debug) {
  841. print STDERR "DEBUG: [dir $_] old path: $oldpath\n";
  842. print STDERR "DEBUG: [dir $_] new base: $absdir\n";
  843. print STDERR "DEBUG: [dir $_] resulting new path: $newpath\n";
  844. }
  845. $tmp_directories{$_} = $newpath;
  846. }
  847. }
  848. # Treat each environment variable that was used to get us the values in
  849. # %directories the same was as the paths in %directories, so any sub
  850. # process can use their values properly as well
  851. foreach (@direnv) {
  852. if (!file_name_is_absolute($ENV{$_})) {
  853. my $oldpath = abs_path($ENV{$_});
  854. my $newpath = abs2rel($oldpath, $absdir);
  855. if ($debug) {
  856. print STDERR "DEBUG: [env $_] old path: $oldpath\n";
  857. print STDERR "DEBUG: [env $_] new base: $absdir\n";
  858. print STDERR "DEBUG: [env $_] resulting new path: $newpath\n";
  859. }
  860. $tmp_ENV{$_} = $newpath;
  861. }
  862. }
  863. # Should we just bail out here as well? I'm unsure.
  864. return undef unless chdir($dir);
  865. # We put back new values carefully. Doing the obvious
  866. # %directories = ( %tmp_directories )
  867. # will clear out any value that happens to be an absolute path
  868. foreach (keys %tmp_directories) {
  869. $directories{$_} = $tmp_directories{$_};
  870. }
  871. foreach (keys %tmp_ENV) {
  872. $ENV{$_} = $tmp_ENV{$_};
  873. }
  874. if ($debug) {
  875. print STDERR "DEBUG: __cwd(), directories and files:\n";
  876. print STDERR " Moving from $abscurdir\n";
  877. print STDERR " Moving to $absdir\n";
  878. print STDERR "\n";
  879. print STDERR " \$directories{BLDTEST} = \"$directories{BLDTEST}\"\n";
  880. print STDERR " \$directories{SRCTEST} = \"$directories{SRCTEST}\"\n";
  881. print STDERR " \$directories{SRCDATA} = \"$directories{SRCDATA}\"\n"
  882. if exists $directories{SRCDATA};
  883. print STDERR " \$directories{RESULTS} = \"$directories{RESULTS}\"\n";
  884. print STDERR " \$directories{BLDAPPS} = \"$directories{BLDAPPS}\"\n";
  885. print STDERR " \$directories{SRCAPPS} = \"$directories{SRCAPPS}\"\n";
  886. print STDERR " \$directories{SRCTOP} = \"$directories{SRCTOP}\"\n";
  887. print STDERR " \$directories{BLDTOP} = \"$directories{BLDTOP}\"\n";
  888. print STDERR "\n";
  889. print STDERR " the way back is \"$reverse\"\n";
  890. }
  891. return $reverse;
  892. }
  893. # __wrap_cmd CMD
  894. # __wrap_cmd CMD, EXE_SHELL
  895. #
  896. # __wrap_cmd "wraps" CMD (string) with a beginning command that makes sure
  897. # the command gets executed with an appropriate environment. If EXE_SHELL
  898. # is given, it is used as the beginning command.
  899. #
  900. # __wrap_cmd returns a list that should be used to build up a larger list
  901. # of command tokens, or be joined together like this:
  902. #
  903. # join(" ", __wrap_cmd($cmd))
  904. sub __wrap_cmd {
  905. my $cmd = shift;
  906. my $exe_shell = shift;
  907. my @prefix = ();
  908. if (defined($exe_shell)) {
  909. # If $exe_shell is defined, trust it
  910. @prefix = ( $exe_shell );
  911. } else {
  912. # Otherwise, use the standard wrapper
  913. my $std_wrapper = __bldtop_file("util", "wrap.pl");
  914. if ($^O eq "VMS" || $^O eq "MSWin32") {
  915. # On VMS and Windows, we run the perl executable explicitly,
  916. # with necessary fixups. We might not need that for Windows,
  917. # but that depends on if the user has associated the '.pl'
  918. # extension with a perl interpreter, so better be safe.
  919. @prefix = ( __fixup_prg($^X), $std_wrapper );
  920. } else {
  921. # Otherwise, we assume Unix semantics, and trust that the #!
  922. # line activates perl for us.
  923. @prefix = ( $std_wrapper );
  924. }
  925. }
  926. return (@prefix, $cmd);
  927. }
  928. # __fixup_prg PROG
  929. #
  930. # __fixup_prg does whatever fixup is needed to execute an executable binary
  931. # given by PROG (string).
  932. #
  933. # __fixup_prg returns a string with the possibly prefixed program path spec.
  934. sub __fixup_prg {
  935. my $prog = shift;
  936. return join(' ', fixup_cmd($prog));
  937. }
  938. # __decorate_cmd NUM, CMDARRAYREF
  939. #
  940. # __decorate_cmd takes a command number NUM and a command token array
  941. # CMDARRAYREF, builds up a command string from them and decorates it
  942. # with necessary redirections.
  943. # __decorate_cmd returns a list of two strings, one with the command
  944. # string to actually be used, the other to be displayed for the user.
  945. # The reason these strings might differ is that we redirect stderr to
  946. # the null device unless we're verbose and unless the user has
  947. # explicitly specified a stderr redirection.
  948. sub __decorate_cmd {
  949. BAIL_OUT("Must run setup() first") if (! $test_name);
  950. my $num = shift;
  951. my $cmd = shift;
  952. my %opts = @_;
  953. my $cmdstr = join(" ", @$cmd);
  954. my $null = devnull();
  955. my $fileornull = sub { $_[0] ? $_[0] : $null; };
  956. my $stdin = "";
  957. my $stdout = "";
  958. my $stderr = "";
  959. my $saved_stderr = undef;
  960. $stdin = " < ".$fileornull->($opts{stdin}) if exists($opts{stdin});
  961. $stdout= " > ".$fileornull->($opts{stdout}) if exists($opts{stdout});
  962. $stderr=" 2> ".$fileornull->($opts{stderr}) if exists($opts{stderr});
  963. my $display_cmd = "$cmdstr$stdin$stdout$stderr";
  964. # VMS program output escapes TAP::Parser
  965. if ($^O eq 'VMS') {
  966. $stderr=" 2> ".$null
  967. unless $stderr || !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
  968. }
  969. $cmdstr .= "$stdin$stdout$stderr";
  970. if ($debug) {
  971. print STDERR "DEBUG[__decorate_cmd]: \$cmdstr = \"$cmdstr\"\n";
  972. print STDERR "DEBUG[__decorate_cmd]: \$display_cmd = \"$display_cmd\"\n";
  973. }
  974. return ($cmdstr, $display_cmd);
  975. }
  976. =head1 SEE ALSO
  977. L<Test::More>, L<Test::Harness>
  978. =head1 AUTHORS
  979. Richard Levitte E<lt>levitte@openssl.orgE<gt> with assistance and
  980. inspiration from Andy Polyakov E<lt>appro@openssl.org<gt>.
  981. =cut
  982. no warnings 'redefine';
  983. sub subtest {
  984. $level++;
  985. Test::More::subtest @_;
  986. $level--;
  987. };
  988. 1;