cijobs.pl 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510
  1. #!/usr/bin/perl
  2. #***************************************************************************
  3. # _ _ ____ _
  4. # Project ___| | | | _ \| |
  5. # / __| | | | |_) | |
  6. # | (__| |_| | _ <| |___
  7. # \___|\___/|_| \_\_____|
  8. #
  9. # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  10. #
  11. # This software is licensed as described in the file COPYING, which
  12. # you should have received as part of this distribution. The terms
  13. # are also available at https://curl.se/docs/copyright.html.
  14. #
  15. # You may opt to use, copy, modify, merge, publish, distribute and/or sell
  16. # copies of the Software, and permit persons to whom the Software is
  17. # furnished to do so, under the terms of the COPYING file.
  18. #
  19. # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  20. # KIND, either express or implied.
  21. #
  22. # SPDX-License-Identifier: curl
  23. #
  24. ###########################################################################
  25. my %filelevel= ('file' => 1,
  26. 'service' => 1);
  27. my $jobid = 1;
  28. sub submit {
  29. my ($jref)=@_;
  30. my %job = %$jref;
  31. printf "\n##### job %u \n", $jobid++;
  32. for my $k (sort keys %job) {
  33. printf "%s: %s\n", $k, $job{$k} if($job{$k});
  34. undef $$jref{$k} if(!$filelevel{$k});
  35. }
  36. }
  37. sub githubactions {
  38. my ($tag)=@_;
  39. my @files= `git ls-tree -r --name-only $tag .github/workflows 2>/dev/null`;
  40. my $c = 0;
  41. foreach my $f (sort @files) {
  42. my $j = 0;
  43. my $m = -1;
  44. my $done = 0;
  45. chomp $f;
  46. open(G, "git show $tag:$f 2>/dev/null|");
  47. # start counting file jobs
  48. undef %job;
  49. $job{'file'} = $f;
  50. $job{'service'} = "gha";
  51. my @cc;
  52. my $os;
  53. my $topname;
  54. my $line = 1;
  55. while(<G>) {
  56. $job{'line'} = $line;
  57. if($_ =~ /^name: (.*)/) {
  58. $topname=$1;
  59. }
  60. elsif($_ =~ /runs-on: (.*)/) {
  61. my $r = $1;
  62. #print "runs-on: $r\n";
  63. if($r =~ /ubuntu/) {
  64. $os = "linux";
  65. }
  66. elsif($r =~ /macos/) {
  67. $os = "macos";
  68. }
  69. elsif($r =~ /windows/) {
  70. $os = "windows";
  71. }
  72. # commit previously counted jobs
  73. $c += $j;
  74. # non-matrix job
  75. $j = 1;
  76. }
  77. elsif($_ =~ /^\s*matrix:/) {
  78. # switch to matrix mode
  79. $m = 0;
  80. $j = 0;
  81. }
  82. elsif($_ =~ /^ - run: .* apt-get install (.*)/) {
  83. $job{'install'} = $1;
  84. }
  85. elsif($m >= 0) {
  86. if($_ =~ /^ - name: (.*)/) {
  87. # matrix job
  88. #print "name: $1\n";
  89. $job{'name'} = $1;
  90. $j += ($m?$m:1);
  91. }
  92. elsif($_ =~ /install: (.*)/) {
  93. $job{'install'} = $1;
  94. }
  95. elsif($_ =~ /( |curl-)configure: (.*)/) {
  96. $job{'configure'} = $2;
  97. $job{'os'}=$os;
  98. submit(\%job);
  99. $done++;
  100. }
  101. elsif($_ =~ /generate: (.*)/) {
  102. $job{'cmake'} = $1;
  103. if($m) {
  104. # matrix mode, multiple copies
  105. my %dupe = %job;
  106. for my $cc (@cc) {
  107. %job = %dupe;
  108. $job{'cc'} = $cc;
  109. $job{'os'}=$os;
  110. submit(\%job);
  111. $done++;
  112. }
  113. }
  114. else {
  115. $job{'os'}=$os;
  116. submit(\%job);
  117. $done++;
  118. }
  119. }
  120. elsif($_ =~ /- CC: (.*)/) {
  121. # matrix multiplier
  122. push @cc, $1;
  123. $m++;
  124. }
  125. elsif($_ =~ /^\s*steps:/) {
  126. # disable matrix mode
  127. $m = -1;
  128. }
  129. }
  130. $line++;
  131. }
  132. close(G);
  133. # commit final counted jobs
  134. $c += $j;
  135. if(!$done) {
  136. $job{'name'} = $topname? $topname : '[unnamed]';
  137. $job{'os'}=$os;
  138. submit(\%job);
  139. $done++;
  140. }
  141. # reset internal job counter
  142. $j = 0;
  143. }
  144. #print "Jobs: $c\n";
  145. return $c;
  146. }
  147. sub azurepipelines {
  148. my ($tag)=@_;
  149. open(G, "git show $tag:.azure-pipelines.yml 2>/dev/null|");
  150. my $c = 0;
  151. my $j = 0;
  152. my $m = -1;
  153. my $image;
  154. my %job;
  155. my $line = 1;
  156. my $os;
  157. $job{'file'} = ".azure-pipelines.yml";
  158. $job{'service'} = "azure";
  159. while(<G>) {
  160. if($_ =~ /^ vmImage: (.*)/) {
  161. my $i = $1;
  162. if($i =~ /ubuntu/) {
  163. $os = "linux";
  164. }
  165. elsif($i =~ /windows/) {
  166. $os = "windows";
  167. }
  168. }
  169. elsif($_ =~ /^- stage: (.*)/) {
  170. my $topname = $1;
  171. if($topname !~ /(windows|linux)/) {
  172. $job{'name'} = $topname;
  173. $job{'line'}=$line;
  174. submit(\%job);
  175. }
  176. }
  177. elsif($_ =~ /job:/) {
  178. # commit previously counted jobs
  179. $c += $j;
  180. # initial value for non-matrix job
  181. $j = 1;
  182. }
  183. elsif($_ =~ /matrix:/) {
  184. # start of new matrix list(!)
  185. $m = 0;
  186. $j = 0;
  187. }
  188. elsif($m >= 0) {
  189. if($_ =~ /^ name: (.*)/) {
  190. # single matrix list entry job
  191. $j++;
  192. $job{'name'} = $1;
  193. }
  194. # azure matrix is a simple list,
  195. # therefore no multiplier needed
  196. elsif($_ =~ /steps:/) {
  197. # disable matrix mode
  198. $m = -1;
  199. }
  200. elsif($_ =~ /^ configure: (.*)/) {
  201. $job{'configure'} = $1;
  202. $job{'line'}=$line;
  203. $job{'os'}=$os;
  204. submit(\%job);
  205. }
  206. }
  207. $line++;
  208. }
  209. close(G);
  210. # commit final counted jobs
  211. $c += $j;
  212. return $c;
  213. }
  214. sub appveyor {
  215. my ($tag)=@_;
  216. open(G, "git show $tag:appveyor.yml 2>/dev/null|");
  217. my $c = 0;
  218. my %job;
  219. my $line=0;
  220. $job{'file'} = "appveyor.yml";
  221. $job{'service'} = "appveyor";
  222. while(<G>) {
  223. $line++;
  224. if($_ =~ /^( - |install)/) {
  225. if($job{'image'}) {
  226. $job{'os'} = "windows";
  227. submit(\%job);
  228. $c++;
  229. }
  230. }
  231. $job{'line'} = $line;
  232. if($_ =~ /^ APPVEYOR_BUILD_WORKER_IMAGE: \"(.*)\"/) {
  233. $job{'image'}= $1;
  234. }
  235. elsif($_ =~ /^ BUILD_SYSTEM: (.*)/) {
  236. $job{'build'} = lc($1);
  237. }
  238. elsif($_ =~ /^ PRJ_GEN: \"(.*)\"/) {
  239. $job{'compiler'} = $1;
  240. }
  241. elsif($_ =~ /^ PRJ_CFG: (.*)/) {
  242. $job{'config'} = $1;
  243. }
  244. elsif($_ =~ /^ OPENSSL: (.*)/) {
  245. $job{'openssl'} = $1 eq "ON" ? "true": "false";
  246. }
  247. elsif($_ =~ /^ SCHANNEL: (.*)/) {
  248. $job{'schannel'} = $1 eq "ON" ? "true": "false";
  249. }
  250. elsif($_ =~ /^ ENABLE_UNICODE: (.*)/) {
  251. $job{'unicode'} = $1 eq "ON" ? "true": "false";
  252. }
  253. elsif($_ =~ /^ HTTP_ONLY: (.*)/) {
  254. $job{'http-only'} = $1 eq "ON" ? "true": "false";
  255. }
  256. elsif($_ =~ /^ TESTING: (.*)/) {
  257. $job{'testing'} = $1 eq "ON" ? "true": "false";
  258. }
  259. elsif($_ =~ /^ SHARED: (.*)/) {
  260. $job{'shared'} = $1 eq "ON" ? "true": "false";
  261. }
  262. elsif($_ =~ /^ TARGET: \"-A (.*)\"/) {
  263. $job{'target'} = $1;
  264. }
  265. }
  266. close(G);
  267. return $c;
  268. }
  269. sub cirrus {
  270. my ($tag)=@_;
  271. open(G, "git show $tag:.cirrus.yml 2>/dev/null|");
  272. my $c = 0;
  273. my %job;
  274. my $line=0;
  275. my $name = 0;
  276. my $os;
  277. $job{'file'} = ".cirrus.yml";
  278. $job{'service'} = "cirrus";
  279. while(<G>) {
  280. $line++;
  281. if($_ =~ /^ ( |-) (name|image_family|image):/) {
  282. $c++;
  283. }
  284. if($_ =~ /^ - name:/) {
  285. if($name) {
  286. $job{'os'} = $os;
  287. $job{'line'} = $line;
  288. submit(\%job);
  289. $name = 0;
  290. }
  291. }
  292. if($_ =~ /^ - name: (.*)/) {
  293. $job{'name'} = $1;
  294. $name = 1;
  295. }
  296. elsif($_ =~ /^ image_family: (.*)/) {
  297. $os = "freebsd";
  298. }
  299. elsif($_ =~ /^windows_task:/) {
  300. $os = "windows";
  301. }
  302. elsif($_ =~ /^ prepare: pacman -S --needed --noconfirm --noprogressbar (.*)/) {
  303. $job{'install'} = $1;
  304. }
  305. elsif($_ =~ /^ configure: (.*)/) {
  306. $job{'configure'} = $1;
  307. }
  308. }
  309. close(G);
  310. if($name) {
  311. $job{'os'} = $os;
  312. $job{'line'} = $line;
  313. submit(\%job);
  314. }
  315. return $c;
  316. }
  317. sub circle {
  318. my ($tag)=@_;
  319. open(G, "git show $tag:.circleci/config.yml 2>/dev/null|");
  320. my $c = 0;
  321. my $wf = 0;
  322. my %job;
  323. my %cmd;
  324. my %configure;
  325. my %target;
  326. my $line=0;
  327. my $cmds;
  328. my $jobs;
  329. my $workflow;
  330. $job{'file'} = ".circleci/config.yml";
  331. $job{'service'} = "circleci";
  332. while(<G>) {
  333. $line++;
  334. if($_ =~ /^commands:/) {
  335. # we record configure lines in this state
  336. $cmds = 1;
  337. }
  338. elsif($cmds) {
  339. if($_ =~ /^ ([^ ]*):/) {
  340. $cmdname = $1;
  341. }
  342. elsif($_ =~ /^ .*.\/configure (.*)/) {
  343. $cmd{$cmdname}=$1;
  344. }
  345. }
  346. if($_ =~ /^jobs:/) {
  347. # we record which job runs with configure here
  348. $jobs = 1;
  349. $cmds = 0;
  350. }
  351. elsif($jobs) {
  352. if($_ =~ /^ ([^ ]*):/) {
  353. $jobname = $1;
  354. }
  355. elsif($_ =~ /^ - (configure.*)/) {
  356. $configure{$jobname}=$1;
  357. }
  358. elsif($_ =~ /^ resource_class: arm.medium/) {
  359. $target{$jobname}="arm";
  360. }
  361. }
  362. if($_ =~ /^workflows:/) {
  363. $wf = 1;
  364. $cmds = 0;
  365. }
  366. elsif($wf) {
  367. if($_ =~ /^ ([^ ]+):/) {
  368. $workflow = $1;
  369. }
  370. elsif($_ =~ /^ - (.*)\n/) {
  371. my $jb = $1;
  372. my $cnfgure = $configure{$jb};
  373. my $trgt = $target{$jb};
  374. $job{'configure'} = $cmd{$cnfgure};
  375. $job{'name' }=$workflow;
  376. $job{'os'} = "linux";
  377. $job{'line'} = $line;
  378. $job{'target'} = $trgt if($trgt);
  379. submit(\%job);
  380. }
  381. if($_ =~ / *jobs:/) {
  382. $c++;
  383. }
  384. }
  385. }
  386. close(G);
  387. return $c;
  388. }
  389. sub zuul {
  390. my ($tag)=@_;
  391. open(G, "git show $tag:zuul.d/jobs.yaml 2>/dev/null|");
  392. my $c = 0;
  393. my %job;
  394. my $line=0;
  395. my $type;
  396. $job{'file'} = "zuul.d/jobs.yaml";
  397. $job{'service'} = "zuul";
  398. while(<G>) {
  399. $line++;
  400. #print "L: ($jobmode / $env) $_";
  401. if($_ =~ /^- job:/) {
  402. $jobmode = 1; # start a new
  403. $type="configure";
  404. }
  405. if($jobmode) {
  406. if($apt) {
  407. if($_ =~ /^ - (.*)/) {
  408. my $value = $1;
  409. $job{'install'} .= "$value ";
  410. }
  411. else {
  412. $apt = 0; # end of curl_apt_packages
  413. }
  414. }
  415. if($env) {
  416. if($envcont) {
  417. if($_ =~ /^ (.*)/) {
  418. $job{$envcont} .= "$1 ";
  419. }
  420. else {
  421. $envcont = "";
  422. }
  423. }
  424. if($_ =~ /^ ([^:]+): (.*)/) {
  425. my ($var, $value) = ($1, $2);
  426. if($var eq "C") {
  427. $var = $type;
  428. }
  429. elsif($var eq "T") {
  430. $var = "tests";
  431. if($value eq "cmake") {
  432. # otherwise it remains configure
  433. $type = "cmake";
  434. }
  435. }
  436. elsif($var eq "CC") {
  437. $var = "compiler";
  438. }
  439. elsif($var eq "CHECKSRC") {
  440. $job{'checksrc'} = $value ? "true": "false";
  441. $var = "";
  442. }
  443. else {
  444. $var = "";
  445. }
  446. if($value eq ">-") {
  447. $envcont = $var;
  448. }
  449. elsif($var) {
  450. $job{$var} = $value;
  451. }
  452. }
  453. elsif($_ !~ /^ /) {
  454. # end of envs
  455. $env = 0;
  456. }
  457. }
  458. if($_ =~ /^ curl_env:/) {
  459. $env = 1; # start of envs
  460. }
  461. elsif($_ =~ /^ curl_apt_packages:/) {
  462. $apt = 1; # start of apt packages
  463. }
  464. elsif($_ =~ /^ name: (.*)/) {
  465. my $n = $1;
  466. if($n eq "curl-base") {
  467. # not counted
  468. $jobmode = 0;
  469. next;
  470. }
  471. $job{'name'} = $n;
  472. }
  473. elsif($_ =~ /^\n\z/) {
  474. # a job is complete
  475. $job{'line'}=$line;
  476. $job{'os'}="linux";
  477. submit(\%job);
  478. $jobmode = 0;
  479. $c++;
  480. }
  481. }
  482. }
  483. close(G);
  484. return $c;
  485. }
  486. my $tag = `git rev-parse --abbrev-ref HEAD 2>/dev/null` || "master";
  487. chomp $tag;
  488. githubactions($tag);
  489. azurepipelines($tag);
  490. appveyor($tag);
  491. zuul($tag);
  492. cirrus($tag);
  493. circle($tag);