proctests.cc 80 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636
  1. #include <cassert>
  2. #include <iostream>
  3. #include <list>
  4. #include <utility>
  5. #include <string>
  6. #include <sstream>
  7. #include "service.h"
  8. #include "proc-service.h"
  9. #include "dinit-util.h"
  10. #include "test_procservice.h"
  11. // Tests of process-service related functionality.
  12. //
  13. // These tests work mostly by completely mocking out the base_process_service class. The mock
  14. // implementations can be found in test-baseproc.cc.
  15. #ifdef NDEBUG
  16. #error "This file must be built with assertions ENABLED!"
  17. #endif
  18. extern eventloop_t event_loop;
  19. constexpr static auto REG = dependency_type::REGULAR;
  20. constexpr static auto WAITS = dependency_type::WAITS_FOR;
  21. // Regular service start
  22. void test_proc_service_start()
  23. {
  24. using namespace std;
  25. service_set sset;
  26. ha_string command = "test-command";
  27. list<pair<unsigned,unsigned>> command_offsets;
  28. command_offsets.emplace_back(0, command.length());
  29. std::list<prelim_dep> depends;
  30. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  31. init_service_defaults(p);
  32. sset.add_service(&p);
  33. p.start();
  34. sset.process_queues();
  35. assert(p.get_state() == service_state_t::STARTING);
  36. base_process_service_test::exec_succeeded(&p);
  37. sset.process_queues();
  38. assert(p.get_state() == service_state_t::STARTED);
  39. assert(event_loop.active_timers.size() == 0);
  40. sset.remove_service(&p);
  41. }
  42. // Test start with readiness notification
  43. void test_proc_notify_start()
  44. {
  45. using namespace std;
  46. service_set sset;
  47. ha_string command = "test-command";
  48. list<pair<unsigned,unsigned>> command_offsets;
  49. command_offsets.emplace_back(0, command.length());
  50. std::list<prelim_dep> depends;
  51. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  52. init_service_defaults(p);
  53. p.set_notification_fd(3);
  54. sset.add_service(&p);
  55. p.start();
  56. sset.process_queues();
  57. assert(p.get_state() == service_state_t::STARTING);
  58. base_process_service_test::exec_succeeded(&p);
  59. sset.process_queues();
  60. assert(p.get_state() == service_state_t::STARTING);
  61. int nfd = base_process_service_test::get_notification_fd(&p);
  62. assert(nfd > 0);
  63. char notifystr[] = "ok started\n";
  64. std::vector<char> rnotifystr;
  65. rnotifystr.insert(rnotifystr.end(), notifystr, notifystr + sizeof(notifystr));
  66. bp_sys::supply_read_data(nfd, std::move(rnotifystr));
  67. event_loop.regd_fd_watchers[nfd]->fd_event(event_loop, nfd, dasynq::IN_EVENTS);
  68. assert(p.get_state() == service_state_t::STARTED);
  69. assert(event_loop.active_timers.size() == 0);
  70. sset.remove_service(&p);
  71. }
  72. // Unexpected termination
  73. void test_proc_unexpected_term()
  74. {
  75. using namespace std;
  76. service_set sset;
  77. ha_string command = "test-command";
  78. list<pair<unsigned,unsigned>> command_offsets;
  79. command_offsets.emplace_back(0, command.length());
  80. std::list<prelim_dep> depends;
  81. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  82. init_service_defaults(p);
  83. sset.add_service(&p);
  84. service_record d1 {&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}}};
  85. sset.add_service(&d1);
  86. d1.start();
  87. sset.process_queues();
  88. base_process_service_test::exec_succeeded(&p);
  89. sset.process_queues();
  90. assert(p.get_state() == service_state_t::STARTED);
  91. assert(p.get_target_state() == service_state_t::STARTED);
  92. assert(d1.get_state() == service_state_t::STARTED);
  93. assert(d1.get_target_state() == service_state_t::STARTED);
  94. base_process_service_test::handle_exit(&p, 0);
  95. assert(p.get_target_state() == service_state_t::STOPPED);
  96. assert(p.get_state() == service_state_t::STOPPED);
  97. assert(p.get_stop_reason() == stopped_reason_t::TERMINATED);
  98. assert(event_loop.active_timers.size() == 0);
  99. assert(d1.get_state() == service_state_t::STOPPED);
  100. assert(d1.get_target_state() == service_state_t::STOPPED);
  101. sset.remove_service(&d1);
  102. sset.remove_service(&p);
  103. }
  104. // Unexpected termination until restarts exhausted, followed by a normal start.
  105. void test_proc_term_start()
  106. {
  107. using namespace std;
  108. service_set sset;
  109. ha_string command = "test-command";
  110. list<pair<unsigned,unsigned>> command_offsets;
  111. command_offsets.emplace_back(0, command.length());
  112. std::list<prelim_dep> depends;
  113. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  114. init_service_defaults(p);
  115. // One restart per 1000 interval
  116. p.set_restart_interval({1000, 0}, 1);
  117. p.set_auto_restart(auto_restart_mode::ALWAYS);
  118. sset.add_service(&p);
  119. // Start the service
  120. p.start();
  121. sset.process_queues();
  122. base_process_service_test::exec_succeeded(&p);
  123. sset.process_queues();
  124. assert(p.get_state() == service_state_t::STARTED);
  125. assert(p.get_target_state() == service_state_t::STARTED);
  126. // Unexpected termination - should restart
  127. base_process_service_test::handle_exit(&p, 0);
  128. sset.process_queues();
  129. assert(p.get_target_state() == service_state_t::STARTED);
  130. assert(p.get_state() == service_state_t::STARTING);
  131. base_process_service_test::exec_succeeded(&p);
  132. sset.process_queues();
  133. assert(p.get_state() == service_state_t::STARTED);
  134. assert(p.get_target_state() == service_state_t::STARTED);
  135. // 2nd unexpected termination - should stop
  136. base_process_service_test::handle_exit(&p, 0);
  137. sset.process_queues();
  138. assert(p.get_target_state() == service_state_t::STOPPED);
  139. assert(p.get_state() == service_state_t::STOPPED);
  140. // explicit restart:
  141. p.start();
  142. sset.process_queues();
  143. base_process_service_test::exec_succeeded(&p);
  144. sset.process_queues();
  145. assert(p.get_state() == service_state_t::STARTED);
  146. assert(p.get_target_state() == service_state_t::STARTED);
  147. // Now, again, one automatic restart should go through if the process terminates
  148. // restart:
  149. base_process_service_test::handle_exit(&p, 0);
  150. sset.process_queues();
  151. assert(p.get_target_state() == service_state_t::STARTED);
  152. assert(p.get_state() == service_state_t::STARTING);
  153. base_process_service_test::exec_succeeded(&p);
  154. sset.process_queues();
  155. assert(p.get_state() == service_state_t::STARTED);
  156. assert(p.get_target_state() == service_state_t::STARTED);
  157. // and stop:
  158. base_process_service_test::handle_exit(&p, 0);
  159. sset.process_queues();
  160. assert(p.get_target_state() == service_state_t::STOPPED);
  161. assert(p.get_state() == service_state_t::STOPPED);
  162. sset.remove_service(&p);
  163. }
  164. // Unexpected termination with restart
  165. void test_proc_term_restart()
  166. {
  167. using namespace std;
  168. service_set sset;
  169. ha_string command = "test-command";
  170. list<pair<unsigned,unsigned>> command_offsets;
  171. command_offsets.emplace_back(0, command.length());
  172. std::list<prelim_dep> depends;
  173. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  174. init_service_defaults(p);
  175. p.set_auto_restart(auto_restart_mode::ALWAYS);
  176. sset.add_service(&p);
  177. p.start();
  178. sset.process_queues();
  179. pid_t first_pid = bp_sys::last_forked_pid;
  180. base_process_service_test::exec_succeeded(&p);
  181. sset.process_queues();
  182. assert(p.get_state() == service_state_t::STARTED);
  183. assert(event_loop.active_timers.size() == 0);
  184. base_process_service_test::handle_exit(&p, 0);
  185. sset.process_queues();
  186. // Starting, restart timer should be armed:
  187. assert(p.get_state() == service_state_t::STARTING);
  188. assert(event_loop.active_timers.size() == 1);
  189. assert(bp_sys::last_forked_pid == first_pid);
  190. event_loop.advance_time(default_restart_interval);
  191. // Startup timer now active:
  192. assert(event_loop.active_timers.size() == 1);
  193. assert(bp_sys::last_forked_pid == (first_pid + 1));
  194. base_process_service_test::exec_succeeded(&p);
  195. sset.process_queues();
  196. assert(p.get_state() == service_state_t::STARTED);
  197. assert(event_loop.active_timers.size() == 0);
  198. sset.remove_service(&p);
  199. }
  200. // Unexpected termination with restart, with dependent
  201. void test_proc_term_restart2()
  202. {
  203. using namespace std;
  204. service_set sset;
  205. ha_string command = "test-command";
  206. list<pair<unsigned,unsigned>> command_offsets;
  207. command_offsets.emplace_back(0, command.length());
  208. std::list<prelim_dep> depends;
  209. service_record b {&sset, "boot"};
  210. sset.add_service(&b);
  211. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  212. init_service_defaults(p);
  213. p.set_auto_restart(auto_restart_mode::ALWAYS);
  214. sset.add_service(&p);
  215. b.add_dep(&p, WAITS);
  216. pid_t first_pid = bp_sys::last_forked_pid;
  217. b.start();
  218. sset.process_queues();
  219. assert(p.get_state() == service_state_t::STARTING);
  220. assert(bp_sys::last_forked_pid == first_pid + 1);
  221. base_process_service_test::exec_succeeded(&p);
  222. sset.process_queues();
  223. assert(p.get_state() == service_state_t::STARTED);
  224. assert(event_loop.active_timers.size() == 0);
  225. assert(bp_sys::last_forked_pid == first_pid + 1);
  226. // simulate process terminating, should then be restarted:
  227. base_process_service_test::handle_exit(&p, 0);
  228. sset.process_queues();
  229. // Starting, restart timer should be armed:
  230. assert(p.get_state() == service_state_t::STARTING);
  231. assert(event_loop.active_timers.size() == 1);
  232. assert(bp_sys::last_forked_pid == first_pid + 1);
  233. event_loop.advance_time(time_val(0, 200000000));
  234. // startup timer will be active:
  235. assert(event_loop.active_timers.size() == 1);
  236. sset.process_queues();
  237. assert(bp_sys::last_forked_pid == first_pid + 2);
  238. base_process_service_test::exec_succeeded(&p);
  239. sset.process_queues();
  240. assert(p.get_state() == service_state_t::STARTED);
  241. assert(event_loop.active_timers.size() == 0);
  242. assert(sset.count_active_services() == 2);
  243. // Request stop, this time it should not restart:
  244. p.stop(true);
  245. sset.process_queues();
  246. base_process_service_test::handle_exit(&p, 0);
  247. sset.process_queues();
  248. assert(p.get_state() == service_state_t::STOPPED);
  249. assert(event_loop.active_timers.size() == 0);
  250. assert(sset.count_active_services() == 1);
  251. assert(bp_sys::last_forked_pid == first_pid + 2);
  252. // simulate terminate dinit
  253. sset.stop_all_services();
  254. sset.process_queues();
  255. assert(p.get_state() == service_state_t::STOPPED);
  256. assert(event_loop.active_timers.size() == 0);
  257. assert(sset.count_active_services() == 0);
  258. sset.remove_service(&p);
  259. sset.remove_service(&b);
  260. }
  261. // Restart due to dependent, after unexpected termination
  262. void test_proc_term_restart3()
  263. {
  264. using namespace std;
  265. service_set sset;
  266. ha_string command = "test-command";
  267. list<pair<unsigned,unsigned>> command_offsets;
  268. command_offsets.emplace_back(0, command.length());
  269. std::list<prelim_dep> depends;
  270. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  271. init_service_defaults(p);
  272. sset.add_service(&p);
  273. service_record d1 {&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}}};
  274. d1.set_auto_restart(auto_restart_mode::ALWAYS);
  275. sset.add_service(&d1);
  276. d1.start();
  277. sset.process_queues();
  278. base_process_service_test::exec_succeeded(&p);
  279. sset.process_queues();
  280. assert(p.get_state() == service_state_t::STARTED);
  281. assert(p.get_target_state() == service_state_t::STARTED);
  282. assert(d1.get_state() == service_state_t::STARTED);
  283. assert(d1.get_target_state() == service_state_t::STARTED);
  284. base_process_service_test::handle_exit(&p, 0);
  285. sset.process_queues();
  286. assert(p.get_target_state() == service_state_t::STARTED);
  287. assert(p.get_state() == service_state_t::STARTING);
  288. assert(p.get_stop_reason() == stopped_reason_t::TERMINATED);
  289. assert(d1.get_state() == service_state_t::STARTING);
  290. assert(d1.get_target_state() == service_state_t::STARTED);
  291. sset.remove_service(&d1);
  292. sset.remove_service(&p);
  293. event_loop.active_timers.clear();
  294. }
  295. // Restart after unexpected termination, start times out
  296. void test_proc_term_restart4()
  297. {
  298. using namespace std;
  299. service_set sset;
  300. ha_string command = "test-command";
  301. list<pair<unsigned,unsigned>> command_offsets;
  302. command_offsets.emplace_back(0, command.length());
  303. std::list<prelim_dep> depends;
  304. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  305. init_service_defaults(p);
  306. p.set_auto_restart(auto_restart_mode::ALWAYS);
  307. time_val start_timeout {5, 0};
  308. p.set_start_timeout(start_timeout);
  309. sset.add_service(&p);
  310. p.start();
  311. sset.process_queues();
  312. pid_t first_pid = bp_sys::last_forked_pid;
  313. base_process_service_test::exec_succeeded(&p);
  314. sset.process_queues();
  315. assert(p.get_state() == service_state_t::STARTED);
  316. assert(event_loop.active_timers.size() == 0);
  317. base_process_service_test::handle_exit(&p, 0);
  318. sset.process_queues();
  319. // Starting, restart timer should be armed:
  320. assert(p.get_state() == service_state_t::STARTING);
  321. assert(event_loop.active_timers.size() == 1);
  322. assert(bp_sys::last_forked_pid == first_pid);
  323. event_loop.advance_time(default_restart_interval);
  324. // restart timer should have stopped, start timeout timer should have started
  325. assert(event_loop.active_timers.size() == 1);
  326. assert(bp_sys::last_forked_pid == (first_pid + 1));
  327. event_loop.advance_time(start_timeout);
  328. assert(p.get_state() == service_state_t::STOPPING);
  329. assert(event_loop.active_timers.size() == 1); // stop timeout should be set
  330. base_process_service_test::exec_succeeded(&p); // the exec must finally succeed...
  331. base_process_service_test::handle_exit(&p, 0);
  332. assert(p.get_state() == service_state_t::STOPPED);
  333. assert(event_loop.active_timers.size() == 0);
  334. sset.remove_service(&p);
  335. }
  336. // Unexpected termination but only restarts on non-zero exit code.
  337. void test_proc_term_restart5()
  338. {
  339. using namespace std;
  340. service_set sset;
  341. ha_string command = "test-command";
  342. list<pair<unsigned,unsigned>> command_offsets;
  343. command_offsets.emplace_back(0, command.length());
  344. std::list<prelim_dep> depends;
  345. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  346. init_service_defaults(p);
  347. p.set_auto_restart(auto_restart_mode::ON_FAILURE);
  348. sset.add_service(&p);
  349. // Start the service
  350. p.start();
  351. sset.process_queues();
  352. base_process_service_test::exec_succeeded(&p);
  353. sset.process_queues();
  354. assert(p.get_state() == service_state_t::STARTED);
  355. assert(p.get_target_state() == service_state_t::STARTED);
  356. // Unexpected termination - should restart because of 1 exit code
  357. base_process_service_test::handle_exit(&p, 1);
  358. sset.process_queues();
  359. assert(p.get_target_state() == service_state_t::STARTED);
  360. assert(p.get_state() == service_state_t::STARTING);
  361. base_process_service_test::exec_succeeded(&p);
  362. sset.process_queues();
  363. assert(p.get_state() == service_state_t::STARTED);
  364. assert(p.get_target_state() == service_state_t::STARTED);
  365. // 2nd unexpected termination - should stop because of 0 exit code
  366. base_process_service_test::handle_exit(&p, 0);
  367. sset.process_queues();
  368. assert(p.get_target_state() == service_state_t::STOPPED);
  369. assert(p.get_state() == service_state_t::STOPPED);
  370. sset.remove_service(&p);
  371. }
  372. // Unexpected termination but only restarts on specific signals.
  373. void test_proc_term_restart6()
  374. {
  375. using namespace std;
  376. service_set sset;
  377. ha_string command = "test-command";
  378. list<pair<unsigned,unsigned>> command_offsets;
  379. command_offsets.emplace_back(0, command.length());
  380. std::list<prelim_dep> depends;
  381. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  382. init_service_defaults(p);
  383. p.set_auto_restart(auto_restart_mode::ON_FAILURE);
  384. sset.add_service(&p);
  385. // Start the service
  386. p.start();
  387. sset.process_queues();
  388. base_process_service_test::exec_succeeded(&p);
  389. sset.process_queues();
  390. assert(p.get_state() == service_state_t::STARTED);
  391. assert(p.get_target_state() == service_state_t::STARTED);
  392. // Unexpected termination - should restart because of SIGQUIT signal
  393. base_process_service_test::handle_signal_exit(&p, SIGQUIT);
  394. sset.process_queues();
  395. assert(p.get_target_state() == service_state_t::STARTED);
  396. assert(p.get_state() == service_state_t::STARTING);
  397. base_process_service_test::exec_succeeded(&p);
  398. sset.process_queues();
  399. assert(p.get_state() == service_state_t::STARTED);
  400. assert(p.get_target_state() == service_state_t::STARTED);
  401. int signals[] = { SIGHUP, SIGINT, SIGUSR1, SIGUSR2, SIGTERM };
  402. for (auto signal: signals) {
  403. // unexpected termination - should stop because of specific signal
  404. base_process_service_test::handle_signal_exit(&p, signal);
  405. sset.process_queues();
  406. assert(p.get_target_state() == service_state_t::STOPPED);
  407. assert(p.get_state() == service_state_t::STOPPED);
  408. // explicit restart
  409. p.start();
  410. sset.process_queues();
  411. base_process_service_test::exec_succeeded(&p);
  412. sset.process_queues();
  413. assert(p.get_state() == service_state_t::STARTED);
  414. assert(p.get_target_state() == service_state_t::STARTED);
  415. }
  416. sset.remove_service(&p);
  417. }
  418. // Failure to restart should propagate to dependent
  419. void test_proc_term_restart_fail()
  420. {
  421. using namespace std;
  422. service_set sset;
  423. ha_string command = "test-command";
  424. list<pair<unsigned,unsigned>> command_offsets;
  425. command_offsets.emplace_back(0, command.length());
  426. std::list<prelim_dep> depends;
  427. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  428. init_service_defaults(p);
  429. p.set_auto_restart(auto_restart_mode::ALWAYS);
  430. sset.add_service(&p);
  431. service_record p_dpt1 {&sset, "dpt1", service_type_t::INTERNAL, {{ &p, REG }}};
  432. p_dpt1.set_auto_restart(auto_restart_mode::ALWAYS);
  433. sset.add_service(&p_dpt1);
  434. service_record p_dpt2 {&sset, "dpt2", service_type_t::INTERNAL, {{ &p_dpt1, WAITS }}};
  435. sset.add_service(&p_dpt2);
  436. p_dpt2.start();
  437. sset.process_queues();
  438. for (int i = 0; i < 3; ++i) {
  439. assert(p.get_state() == service_state_t::STARTING);
  440. base_process_service_test::exec_succeeded(&p);
  441. sset.process_queues();
  442. assert(p.get_state() == service_state_t::STARTED);
  443. assert(event_loop.active_timers.size() == 0);
  444. base_process_service_test::handle_exit(&p, 0);
  445. sset.process_queues();
  446. // Starting, restart timer should be armed:
  447. assert(p.get_state() == service_state_t::STARTING);
  448. assert(event_loop.active_timers.size() == 1);
  449. event_loop.advance_time(default_restart_interval);
  450. }
  451. assert(p.get_state() == service_state_t::STARTING);
  452. base_process_service_test::exec_succeeded(&p);
  453. sset.process_queues();
  454. assert(p.get_state() == service_state_t::STARTED);
  455. assert(event_loop.active_timers.size() == 0);
  456. // There should be no attempt to restart this time:
  457. base_process_service_test::handle_exit(&p, 0);
  458. sset.process_queues();
  459. assert(event_loop.active_timers.size() == 0);
  460. assert(p.get_state() == service_state_t::STOPPED);
  461. assert(p_dpt1.get_state() == service_state_t::STOPPED);
  462. sset.remove_service(&p_dpt2);
  463. sset.remove_service(&p_dpt1);
  464. sset.remove_service(&p);
  465. }
  466. // Termination via stop request
  467. void test_term_via_stop()
  468. {
  469. using namespace std;
  470. service_set sset;
  471. ha_string command = "test-command";
  472. list<pair<unsigned,unsigned>> command_offsets;
  473. command_offsets.emplace_back(0, command.length());
  474. std::list<prelim_dep> depends;
  475. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  476. init_service_defaults(p);
  477. sset.add_service(&p);
  478. p.start();
  479. sset.process_queues();
  480. base_process_service_test::exec_succeeded(&p);
  481. sset.process_queues();
  482. assert(p.get_state() == service_state_t::STARTED);
  483. assert(event_loop.active_timers.size() == 0);
  484. p.stop(true);
  485. sset.process_queues();
  486. assert(p.get_state() == service_state_t::STOPPING);
  487. assert(event_loop.active_timers.size() == 1);
  488. base_process_service_test::handle_exit(&p, 0);
  489. sset.process_queues();
  490. assert(p.get_state() == service_state_t::STOPPED);
  491. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  492. assert(event_loop.active_timers.size() == 0);
  493. sset.remove_service(&p);
  494. }
  495. // Termination via stop request, ensure reason is reset:
  496. void test_term_via_stop2()
  497. {
  498. using namespace std;
  499. service_set sset;
  500. ha_string command = "test-command";
  501. list<pair<unsigned,unsigned>> command_offsets;
  502. command_offsets.emplace_back(0, command.length());
  503. std::list<prelim_dep> depends;
  504. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  505. init_service_defaults(p);
  506. sset.add_service(&p);
  507. p.start();
  508. sset.process_queues();
  509. // first set it up with failure reason:
  510. base_process_service_test::exec_failed(&p, ENOENT);
  511. sset.process_queues();
  512. assert(p.get_state() == service_state_t::STOPPED);
  513. assert(p.get_stop_reason() == stopped_reason_t::EXECFAILED);
  514. // now restart clean:
  515. p.start();
  516. sset.process_queues();
  517. base_process_service_test::exec_succeeded(&p);
  518. sset.process_queues();
  519. assert(p.get_state() == service_state_t::STARTED);
  520. assert(event_loop.active_timers.size() == 0);
  521. p.stop(true);
  522. sset.process_queues();
  523. assert(p.get_state() == service_state_t::STOPPING);
  524. base_process_service_test::handle_exit(&p, 0);
  525. sset.process_queues();
  526. assert(p.get_state() == service_state_t::STOPPED);
  527. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  528. assert(event_loop.active_timers.size() == 0);
  529. sset.remove_service(&p);
  530. }
  531. // stop twice
  532. void test_term_via_stop3()
  533. {
  534. using namespace std;
  535. service_set sset;
  536. ha_string command = "test-command";
  537. list<pair<unsigned,unsigned>> command_offsets;
  538. command_offsets.emplace_back(0, command.length());
  539. std::list<prelim_dep> depends;
  540. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  541. init_service_defaults(p);
  542. sset.add_service(&p);
  543. p.start();
  544. sset.process_queues();
  545. base_process_service_test::exec_succeeded(&p);
  546. sset.process_queues();
  547. assert(p.get_state() == service_state_t::STARTED);
  548. assert(event_loop.active_timers.size() == 0);
  549. p.stop(true);
  550. sset.process_queues();
  551. assert(p.get_state() == service_state_t::STOPPING);
  552. assert(event_loop.active_timers.size() == 1);
  553. base_process_service_test::handle_exit(&p, 0);
  554. sset.process_queues();
  555. assert(p.get_state() == service_state_t::STOPPED);
  556. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  557. assert(event_loop.active_timers.size() == 0);
  558. p.start();
  559. sset.process_queues();
  560. base_process_service_test::exec_succeeded(&p);
  561. sset.process_queues();
  562. assert(p.get_state() == service_state_t::STARTED);
  563. assert(event_loop.active_timers.size() == 0);
  564. bp_sys::last_sig_sent = 0; // make sure signal is re-sent
  565. p.stop(true);
  566. sset.process_queues();
  567. assert(p.get_state() == service_state_t::STOPPING);
  568. assert(event_loop.active_timers.size() == 1);
  569. assert(bp_sys::last_sig_sent == SIGTERM);
  570. base_process_service_test::handle_exit(&p, 0);
  571. sset.process_queues();
  572. assert(p.get_state() == service_state_t::STOPPED);
  573. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  574. assert(event_loop.active_timers.size() == 0);
  575. sset.remove_service(&p);
  576. }
  577. // Time-out during start
  578. void test_proc_start_timeout()
  579. {
  580. using namespace std;
  581. service_set sset;
  582. ha_string command = "test-command";
  583. list<pair<unsigned,unsigned>> command_offsets;
  584. command_offsets.emplace_back(0, command.length());
  585. std::list<prelim_dep> depends;
  586. scripted_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  587. init_service_defaults(p);
  588. p.set_start_timeout(time_val(10,0));
  589. sset.add_service(&p);
  590. p.start();
  591. sset.process_queues();
  592. assert(p.get_state() == service_state_t::STARTING);
  593. event_loop.advance_time(time_val(10,0));
  594. sset.process_queues();
  595. assert(p.get_state() == service_state_t::STOPPING);
  596. base_process_service_test::handle_signal_exit(&p, SIGTERM);
  597. sset.process_queues();
  598. // We set no stop script, so state should now be STOPPED with no timer set
  599. assert(p.get_state() == service_state_t::STOPPED);
  600. assert(p.get_stop_reason() == stopped_reason_t::TIMEDOUT);
  601. assert(event_loop.active_timers.size() == 0);
  602. sset.remove_service(&p);
  603. }
  604. // Test that a timeout doesn't stop a "waits for" dependent to fail to start
  605. void test_proc_start_timeout2()
  606. {
  607. using namespace std;
  608. service_set sset;
  609. ha_string command = "test-command";
  610. list<pair<unsigned,unsigned>> command_offsets;
  611. command_offsets.emplace_back(0, command.length());
  612. std::list<prelim_dep> depends;
  613. scripted_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  614. p.set_start_timeout(time_val {1,0});
  615. init_service_defaults(p);
  616. sset.add_service(&p);
  617. service_record ts {&sset, "test-service-1", service_type_t::INTERNAL,
  618. {{&p, dependency_type::WAITS_FOR}} };
  619. ts.start();
  620. sset.process_queues();
  621. assert(p.get_state() == service_state_t::STARTING);
  622. assert(ts.get_state() == service_state_t::STARTING);
  623. event_loop.advance_time(time_val {1,0}); // start timer should expire
  624. sset.process_queues();
  625. assert(p.get_state() == service_state_t::STOPPING);
  626. base_process_service_test::handle_exit(&p, 0);
  627. sset.process_queues();
  628. assert(p.get_state() == service_state_t::STOPPED);
  629. assert(p.get_stop_reason() == stopped_reason_t::TIMEDOUT);
  630. assert(ts.get_state() == service_state_t::STARTED);
  631. assert(event_loop.active_timers.size() == 0);
  632. sset.remove_service(&p);
  633. }
  634. // Test exec() failure for process service start.
  635. void test_proc_start_execfail()
  636. {
  637. using namespace std;
  638. service_set sset;
  639. ha_string command = "test-command";
  640. list<pair<unsigned,unsigned>> command_offsets;
  641. command_offsets.emplace_back(0, command.length());
  642. std::list<prelim_dep> depends;
  643. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  644. init_service_defaults(p);
  645. sset.add_service(&p);
  646. p.start();
  647. sset.process_queues();
  648. assert(p.get_state() == service_state_t::STARTING);
  649. base_process_service_test::exec_failed(&p, ENOENT);
  650. sset.process_queues();
  651. assert(p.get_state() == service_state_t::STOPPED);
  652. assert(p.get_stop_reason() == stopped_reason_t::EXECFAILED);
  653. assert(event_loop.active_timers.size() == 0);
  654. sset.remove_service(&p);
  655. }
  656. // Test no ready notification before process terminates
  657. void test_proc_notify_fail()
  658. {
  659. using namespace std;
  660. service_set sset;
  661. ha_string command = "test-command";
  662. list<pair<unsigned,unsigned>> command_offsets;
  663. command_offsets.emplace_back(0, command.length());
  664. std::list<prelim_dep> depends;
  665. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  666. init_service_defaults(p);
  667. p.set_notification_fd(3);
  668. sset.add_service(&p);
  669. p.start();
  670. sset.process_queues();
  671. assert(p.get_state() == service_state_t::STARTING);
  672. base_process_service_test::exec_succeeded(&p);
  673. sset.process_queues();
  674. assert(p.get_state() == service_state_t::STARTING);
  675. int nfd = base_process_service_test::get_notification_fd(&p);
  676. assert(nfd > 0);
  677. // Signal EOF on notify fd:
  678. event_loop.regd_fd_watchers[nfd]->fd_event(event_loop, nfd, dasynq::IN_EVENTS);
  679. assert(p.get_state() == service_state_t::STOPPING);
  680. base_process_service_test::handle_exit(&p, 0);
  681. sset.process_queues();
  682. assert(p.get_state() == service_state_t::STOPPED);
  683. assert(event_loop.active_timers.size() == 0);
  684. sset.remove_service(&p);
  685. }
  686. // Test stop timeout
  687. void test_proc_stop_timeout()
  688. {
  689. using namespace std;
  690. service_set sset;
  691. ha_string command = "test-command";
  692. list<pair<unsigned,unsigned>> command_offsets;
  693. command_offsets.emplace_back(0, command.length());
  694. std::list<prelim_dep> depends;
  695. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  696. init_service_defaults(p);
  697. p.set_stop_timeout(time_val {10, 0});
  698. sset.add_service(&p);
  699. p.start();
  700. sset.process_queues();
  701. assert(p.get_state() == service_state_t::STARTING);
  702. base_process_service_test::exec_succeeded(&p);
  703. sset.process_queues();
  704. assert(p.get_state() == service_state_t::STARTED);
  705. p.stop(true);
  706. sset.process_queues();
  707. assert(p.get_state() == service_state_t::STOPPING);
  708. assert(bp_sys::last_sig_sent == SIGTERM);
  709. event_loop.advance_time(time_val {10, 0}); // expire stop timer
  710. sset.process_queues();
  711. // kill signal (SIGKILL) should have been sent; process not dead until it's dead, however
  712. assert(p.get_state() == service_state_t::STOPPING);
  713. assert(bp_sys::last_sig_sent == SIGKILL);
  714. base_process_service_test::handle_exit(&p, 0);
  715. sset.process_queues();
  716. assert(p.get_state() == service_state_t::STOPPED);
  717. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  718. assert(event_loop.active_timers.size() == 0);
  719. sset.remove_service(&p);
  720. }
  721. // Smooth recovery
  722. void test_proc_smooth_recovery1()
  723. {
  724. using namespace std;
  725. service_set sset;
  726. ha_string command = "test-command";
  727. list<pair<unsigned,unsigned>> command_offsets;
  728. command_offsets.emplace_back(0, command.length());
  729. std::list<prelim_dep> depends;
  730. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  731. init_service_defaults(p);
  732. p.set_smooth_recovery(true);
  733. p.set_restart_delay(time_val {0, 1000});
  734. sset.add_service(&p);
  735. p.start();
  736. sset.process_queues();
  737. base_process_service_test::exec_succeeded(&p);
  738. sset.process_queues();
  739. pid_t first_instance = bp_sys::last_forked_pid;
  740. assert(p.get_state() == service_state_t::STARTED);
  741. base_process_service_test::handle_exit(&p, 0);
  742. sset.process_queues();
  743. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  744. assert(first_instance == bp_sys::last_forked_pid);
  745. assert(p.get_state() == service_state_t::STARTED);
  746. event_loop.advance_time(time_val {0, 1000});
  747. sset.process_queues();
  748. // Now a new process should've been launched:
  749. assert(first_instance + 1 == bp_sys::last_forked_pid);
  750. assert(p.get_state() == service_state_t::STARTED);
  751. base_process_service_test::exec_succeeded(&p);
  752. sset.process_queues();
  753. assert(event_loop.active_timers.size() == 0);
  754. sset.remove_service(&p);
  755. }
  756. // Smooth recovery without restart delay
  757. void test_proc_smooth_recovery2()
  758. {
  759. using namespace std;
  760. service_set sset;
  761. ha_string command = "test-command";
  762. list<pair<unsigned,unsigned>> command_offsets;
  763. command_offsets.emplace_back(0, command.length());
  764. std::list<prelim_dep> depends;
  765. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  766. init_service_defaults(p);
  767. p.set_smooth_recovery(true);
  768. p.set_restart_delay(time_val(0, 0));
  769. sset.add_service(&p);
  770. p.start();
  771. sset.process_queues();
  772. base_process_service_test::exec_succeeded(&p);
  773. sset.process_queues();
  774. pid_t first_instance = bp_sys::last_forked_pid;
  775. assert(p.get_state() == service_state_t::STARTED);
  776. assert(event_loop.active_timers.size() == 0);
  777. base_process_service_test::handle_exit(&p, 0);
  778. sset.process_queues();
  779. // no restart delay, process should restart immediately:
  780. assert(first_instance + 1 == bp_sys::last_forked_pid);
  781. assert(p.get_state() == service_state_t::STARTED);
  782. base_process_service_test::exec_succeeded(&p);
  783. sset.process_queues();
  784. assert(event_loop.active_timers.size() == 0);
  785. sset.remove_service(&p);
  786. }
  787. // failure during smooth recovery is non-recoverable
  788. void test_proc_smooth_recovery3()
  789. {
  790. using namespace std;
  791. service_set sset;
  792. ha_string command = "test-command";
  793. list<pair<unsigned,unsigned>> command_offsets;
  794. command_offsets.emplace_back(0, command.length());
  795. std::list<prelim_dep> depends;
  796. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  797. init_service_defaults(p);
  798. p.set_smooth_recovery(true);
  799. p.set_restart_delay(time_val(0, 0));
  800. sset.add_service(&p);
  801. service_record d1 {&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}}};
  802. d1.set_auto_restart(auto_restart_mode::ALWAYS);
  803. sset.add_service(&d1);
  804. d1.start();
  805. //p.start();
  806. sset.process_queues();
  807. base_process_service_test::exec_succeeded(&p);
  808. sset.process_queues();
  809. pid_t first_instance = bp_sys::last_forked_pid;
  810. assert(p.get_state() == service_state_t::STARTED);
  811. assert(event_loop.active_timers.size() == 0);
  812. base_process_service_test::handle_exit(&p, 0);
  813. sset.process_queues();
  814. // no restart delay, process should attempt restart immediately:
  815. assert(first_instance + 1 == bp_sys::last_forked_pid);
  816. assert(p.get_state() == service_state_t::STARTED);
  817. assert(event_loop.active_timers.size() == 1); // (restart timer)
  818. base_process_service_test::exec_failed(&p, ENOENT);
  819. sset.process_queues();
  820. assert(p.get_state() == service_state_t::STOPPED);
  821. assert(p.get_target_state() == service_state_t::STOPPED);
  822. assert(d1.get_state() == service_state_t::STOPPED);
  823. assert(d1.get_target_state() == service_state_t::STOPPED);
  824. assert(event_loop.active_timers.size() == 0);
  825. sset.remove_service(&d1);
  826. sset.remove_service(&p);
  827. }
  828. // stop issued during smooth recovery (waiting for restart timer)
  829. void test_proc_smooth_recovery4()
  830. {
  831. using namespace std;
  832. service_set sset;
  833. ha_string command = "test-command";
  834. list<pair<unsigned,unsigned>> command_offsets;
  835. command_offsets.emplace_back(0, command.length());
  836. std::list<prelim_dep> depends;
  837. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  838. init_service_defaults(p);
  839. p.set_smooth_recovery(true);
  840. p.set_restart_delay(time_val(2, 0)); // 2 second restart delay
  841. sset.add_service(&p);
  842. p.start();
  843. sset.process_queues();
  844. base_process_service_test::exec_succeeded(&p);
  845. sset.process_queues();
  846. pid_t first_instance = bp_sys::last_forked_pid;
  847. assert(p.get_state() == service_state_t::STARTED);
  848. assert(event_loop.active_timers.size() == 0);
  849. base_process_service_test::handle_exit(&p, 0);
  850. sset.process_queues();
  851. // smooth recovery should have begun
  852. event_loop.advance_time(time_val(1, 0));
  853. assert(p.get_state() == service_state_t::STARTED);
  854. // If we stop now, timer should be cancelled, no signal should be sent
  855. bp_sys::last_sig_sent = -1;
  856. p.stop(true);
  857. sset.process_queues();
  858. assert(p.get_state() == service_state_t::STOPPED);
  859. assert(first_instance == bp_sys::last_forked_pid); // no more processes launched
  860. assert(bp_sys::last_sig_sent == -1);
  861. assert(event_loop.active_timers.size() == 0);
  862. sset.remove_service(&p);
  863. }
  864. // stop during smooth recovery (waiting for process startup)
  865. void test_proc_smooth_recovery5()
  866. {
  867. using namespace std;
  868. service_set sset;
  869. ha_string command = "test-command";
  870. list<pair<unsigned,unsigned>> command_offsets;
  871. command_offsets.emplace_back(0, command.length());
  872. std::list<prelim_dep> depends;
  873. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  874. init_service_defaults(p);
  875. p.set_smooth_recovery(true);
  876. p.set_restart_delay(time_val {0, 1000});
  877. sset.add_service(&p);
  878. p.start();
  879. sset.process_queues();
  880. base_process_service_test::exec_succeeded(&p);
  881. sset.process_queues();
  882. pid_t first_instance = bp_sys::last_forked_pid;
  883. assert(p.get_state() == service_state_t::STARTED);
  884. base_process_service_test::handle_exit(&p, 0);
  885. sset.process_queues();
  886. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  887. assert(first_instance == bp_sys::last_forked_pid);
  888. assert(p.get_state() == service_state_t::STARTED);
  889. event_loop.advance_time(time_val {0, 1000});
  890. sset.process_queues();
  891. // Now a new process should've been launched:
  892. assert(first_instance + 1 == bp_sys::last_forked_pid);
  893. assert(p.get_state() == service_state_t::STARTED);
  894. // However, at this stage the exec has not succeeded. If we issue a stop, we shouldn't see a signal sent yet,
  895. // since it's not clear what signal to send (term signal might not be SIGTERM, but if it's something else, the
  896. // process before exec() may not respond correctly)
  897. bp_sys::last_sig_sent = -1;
  898. p.stop(true);
  899. assert(bp_sys::last_sig_sent == -1);
  900. assert(p.get_state() == service_state_t::STOPPING);
  901. // Once the exec succeeds, then we should:
  902. base_process_service_test::exec_succeeded(&p);
  903. sset.process_queues();
  904. assert(bp_sys::last_sig_sent == SIGTERM);
  905. base_process_service_test::handle_exit(&p, 0);
  906. sset.process_queues();
  907. assert(p.get_state() == service_state_t::STOPPED);
  908. assert(event_loop.active_timers.size() == 0);
  909. sset.remove_service(&p);
  910. }
  911. // smooth recovery: timeout waiting for readiness notification
  912. void test_proc_smooth_recovery6()
  913. {
  914. using namespace std;
  915. service_set sset;
  916. ha_string command = "test-command";
  917. list<pair<unsigned,unsigned>> command_offsets;
  918. command_offsets.emplace_back(0, command.length());
  919. std::list<prelim_dep> depends;
  920. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  921. init_service_defaults(p);
  922. p.set_smooth_recovery(true);
  923. p.set_restart_delay(time_val {0, 1000});
  924. p.set_start_timeout(time_val {1, 0});
  925. p.set_notification_fd(3);
  926. sset.add_service(&p);
  927. p.start();
  928. sset.process_queues();
  929. base_process_service_test::exec_succeeded(&p);
  930. sset.process_queues();
  931. // readiness notification from process:
  932. int nfd = base_process_service_test::get_notification_fd(&p);
  933. char notifystr[] = "ok started\n";
  934. std::vector<char> rnotifystr;
  935. rnotifystr.insert(rnotifystr.end(), notifystr, notifystr + sizeof(notifystr));
  936. bp_sys::supply_read_data(nfd, std::move(rnotifystr));
  937. event_loop.regd_fd_watchers[nfd]->fd_event(event_loop, nfd, dasynq::IN_EVENTS);
  938. assert(p.get_state() == service_state_t::STARTED);
  939. assert(event_loop.active_timers.size() == 0);
  940. pid_t first_instance = bp_sys::last_forked_pid;
  941. assert(p.get_state() == service_state_t::STARTED);
  942. base_process_service_test::handle_exit(&p, 0);
  943. sset.process_queues();
  944. event_loop.advance_time(time_val {0, 1000});
  945. // new process should've been forked:
  946. assert(first_instance != bp_sys::last_forked_pid);
  947. assert(p.get_state() == service_state_t::STARTED);
  948. assert(p.get_pid() == bp_sys::last_forked_pid);
  949. base_process_service_test::exec_succeeded(&p);
  950. // Now, timeout while waiting for readiness:
  951. bp_sys::last_sig_sent = -1;
  952. event_loop.advance_time(time_val {1, 0});
  953. // We should see the process has been signalled:
  954. assert(bp_sys::last_sig_sent == SIGINT);
  955. assert(p.get_state() == service_state_t::STOPPING);
  956. base_process_service_test::handle_exit(&p, 1);
  957. sset.process_queues();
  958. // The state should now be stopped:
  959. assert(p.get_state() == service_state_t::STOPPED);
  960. assert(event_loop.active_timers.size() == 0);
  961. sset.remove_service(&p);
  962. }
  963. // smooth recovery: termination while waiting for readiness notification
  964. void test_proc_smooth_recovery6a()
  965. {
  966. using namespace std;
  967. service_set sset;
  968. ha_string command = "test-command";
  969. list<pair<unsigned,unsigned>> command_offsets;
  970. command_offsets.emplace_back(0, command.length());
  971. std::list<prelim_dep> depends;
  972. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  973. init_service_defaults(p);
  974. p.set_smooth_recovery(true);
  975. p.set_restart_delay(time_val {0, 1000});
  976. p.set_start_timeout(time_val {1, 0});
  977. p.set_notification_fd(3);
  978. sset.add_service(&p);
  979. p.start();
  980. sset.process_queues();
  981. base_process_service_test::exec_succeeded(&p);
  982. sset.process_queues();
  983. // readiness notification from process:
  984. int nfd = base_process_service_test::get_notification_fd(&p);
  985. char notifystr[] = "ok started\n";
  986. std::vector<char> rnotifystr;
  987. rnotifystr.insert(rnotifystr.end(), notifystr, notifystr + sizeof(notifystr));
  988. bp_sys::supply_read_data(nfd, std::move(rnotifystr));
  989. event_loop.regd_fd_watchers[nfd]->fd_event(event_loop, nfd, dasynq::IN_EVENTS);
  990. assert(p.get_state() == service_state_t::STARTED);
  991. assert(event_loop.active_timers.size() == 0);
  992. pid_t first_instance = bp_sys::last_forked_pid;
  993. assert(p.get_state() == service_state_t::STARTED);
  994. base_process_service_test::handle_exit(&p, 0);
  995. sset.process_queues();
  996. event_loop.advance_time(time_val {0, 1000});
  997. // new process should've been forked:
  998. assert(first_instance != bp_sys::last_forked_pid);
  999. assert(p.get_state() == service_state_t::STARTED);
  1000. assert(p.get_pid() == bp_sys::last_forked_pid);
  1001. base_process_service_test::exec_succeeded(&p);
  1002. sset.process_queues();
  1003. base_process_service_test::handle_exit(&p, 1);
  1004. sset.process_queues();
  1005. // The state should now be stopped:
  1006. assert(p.get_state() == service_state_t::STOPPED);
  1007. assert(event_loop.active_timers.size() == 0);
  1008. sset.remove_service(&p);
  1009. }
  1010. // simulate the launcher process forking a daemon process, and supply the process ID of that
  1011. // daemon process in a pid file.
  1012. static void supply_pid_contents(const char *pid_file, pid_t *daemon_instance_p = nullptr)
  1013. {
  1014. using namespace std;
  1015. pid_t daemon_instance = ++bp_sys::last_forked_pid;
  1016. // Set up the pid file content with the pid of the daemon
  1017. stringstream str;
  1018. str << daemon_instance << std::flush;
  1019. string pid_file_content = str.str();
  1020. vector<char> pid_file_content_v(pid_file_content.begin(), pid_file_content.end());
  1021. bp_sys::supply_file_content(pid_file, std::move(pid_file_content_v));
  1022. if (daemon_instance_p != nullptr) {
  1023. *daemon_instance_p = daemon_instance;
  1024. }
  1025. }
  1026. void test_bgproc_start()
  1027. {
  1028. using namespace std;
  1029. service_set sset;
  1030. ha_string command = "test-command";
  1031. list<pair<unsigned,unsigned>> command_offsets;
  1032. command_offsets.emplace_back(0, command.length());
  1033. std::list<prelim_dep> depends;
  1034. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1035. init_service_defaults(p);
  1036. p.set_pid_file("/run/daemon.pid");
  1037. sset.add_service(&p);
  1038. p.start();
  1039. sset.process_queues();
  1040. assert(p.get_state() == service_state_t::STARTING);
  1041. base_process_service_test::exec_succeeded(&p);
  1042. sset.process_queues();
  1043. assert(p.get_state() == service_state_t::STARTING);
  1044. supply_pid_contents("/run/daemon.pid");
  1045. base_process_service_test::handle_exit(&p, 0);
  1046. assert(p.get_state() == service_state_t::STARTED);
  1047. assert(event_loop.active_timers.size() == 0);
  1048. sset.remove_service(&p);
  1049. }
  1050. void test_bgproc_start_fail()
  1051. {
  1052. using namespace std;
  1053. service_set sset;
  1054. ha_string command = "test-command";
  1055. list<pair<unsigned,unsigned>> command_offsets;
  1056. command_offsets.emplace_back(0, command.length());
  1057. std::list<prelim_dep> depends;
  1058. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1059. init_service_defaults(p);
  1060. p.set_pid_file("/run/daemon.pid");
  1061. sset.add_service(&p);
  1062. p.start();
  1063. sset.process_queues();
  1064. assert(p.get_state() == service_state_t::STARTING);
  1065. base_process_service_test::exec_succeeded(&p);
  1066. sset.process_queues();
  1067. assert(p.get_state() == service_state_t::STARTING);
  1068. base_process_service_test::handle_exit(&p, 0x1); // fail status
  1069. assert(p.get_state() == service_state_t::STOPPED);
  1070. assert(event_loop.active_timers.size() == 0);
  1071. sset.remove_service(&p);
  1072. }
  1073. void test_bgproc_start_fail_pid()
  1074. {
  1075. using namespace std;
  1076. service_set sset;
  1077. ha_string command = "test-command";
  1078. list<pair<unsigned,unsigned>> command_offsets;
  1079. command_offsets.emplace_back(0, command.length());
  1080. std::list<prelim_dep> depends;
  1081. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1082. init_service_defaults(p);
  1083. p.set_pid_file("/run/no-exist-daemon.pid");
  1084. sset.add_service(&p);
  1085. p.start();
  1086. sset.process_queues();
  1087. assert(p.get_state() == service_state_t::STARTING);
  1088. base_process_service_test::exec_succeeded(&p);
  1089. sset.process_queues();
  1090. assert(p.get_state() == service_state_t::STARTING);
  1091. // launcher returns success, but no pid file exists:
  1092. base_process_service_test::handle_exit(&p, 0x0);
  1093. assert(p.get_state() == service_state_t::STOPPED);
  1094. assert(event_loop.active_timers.size() == 0);
  1095. sset.remove_service(&p);
  1096. }
  1097. void test_bgproc_unexpected_term()
  1098. {
  1099. using namespace std;
  1100. service_set sset;
  1101. ha_string command = "test-command";
  1102. list<pair<unsigned,unsigned>> command_offsets;
  1103. command_offsets.emplace_back(0, command.length());
  1104. std::list<prelim_dep> depends;
  1105. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1106. init_service_defaults(p);
  1107. p.set_pid_file("/run/daemon.pid");
  1108. sset.add_service(&p);
  1109. service_record d1 {&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}}};
  1110. sset.add_service(&d1);
  1111. d1.start();
  1112. sset.process_queues();
  1113. base_process_service_test::exec_succeeded(&p);
  1114. assert(p.get_state() == service_state_t::STARTING);
  1115. assert(d1.get_state() == service_state_t::STARTING);
  1116. supply_pid_contents("/run/daemon.pid");
  1117. base_process_service_test::handle_exit(&p, 0);
  1118. assert(p.get_target_state() == service_state_t::STARTED);
  1119. assert(p.get_state() == service_state_t::STARTED);
  1120. assert(d1.get_target_state() == service_state_t::STARTED);
  1121. assert(d1.get_state() == service_state_t::STARTED);
  1122. // Now the backgrounded daemon itself terminates:
  1123. base_process_service_test::handle_exit(&p, 0);
  1124. assert(p.get_state() == service_state_t::STOPPED);
  1125. assert(p.get_target_state() == service_state_t::STOPPED);
  1126. assert(p.get_stop_reason() == stopped_reason_t::TERMINATED);
  1127. assert(event_loop.active_timers.size() == 0);
  1128. assert(d1.get_state() == service_state_t::STOPPED);
  1129. assert(d1.get_target_state() == service_state_t::STOPPED);
  1130. sset.remove_service(&d1);
  1131. sset.remove_service(&p);
  1132. }
  1133. void test_bgproc_smooth_recover()
  1134. {
  1135. using namespace std;
  1136. service_set sset;
  1137. ha_string command = "test-command";
  1138. list<pair<unsigned,unsigned>> command_offsets;
  1139. command_offsets.emplace_back(0, command.length());
  1140. std::list<prelim_dep> depends;
  1141. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1142. init_service_defaults(p);
  1143. p.set_smooth_recovery(true);
  1144. p.set_restart_delay(time_val {0, 1000});
  1145. p.set_pid_file("/run/daemon.pid");
  1146. sset.add_service(&p);
  1147. p.start();
  1148. sset.process_queues();
  1149. base_process_service_test::exec_succeeded(&p);
  1150. sset.process_queues();
  1151. pid_t daemon_instance;
  1152. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1153. assert(p.get_state() == service_state_t::STARTING);
  1154. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1155. sset.process_queues();
  1156. // daemon process has been started now, state should be STARTED
  1157. assert(p.get_state() == service_state_t::STARTED);
  1158. assert(daemon_instance == bp_sys::last_forked_pid);
  1159. base_process_service_test::handle_exit(&p, 0); // exit the daemon process
  1160. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  1161. assert(p.get_state() == service_state_t::STARTED);
  1162. assert(daemon_instance == bp_sys::last_forked_pid);
  1163. event_loop.advance_time(time_val {0, 1000});
  1164. sset.process_queues();
  1165. // Now a new process should've been launched:
  1166. assert(daemon_instance + 1 == bp_sys::last_forked_pid);
  1167. assert(p.get_state() == service_state_t::STARTED);
  1168. assert(event_loop.active_timers.size() == 1); // start timer
  1169. base_process_service_test::exec_succeeded(&p);
  1170. sset.process_queues();
  1171. assert(event_loop.active_timers.size() == 1);
  1172. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1173. base_process_service_test::handle_exit(&p, 0);
  1174. sset.process_queues();
  1175. assert(p.get_state() == service_state_t::STARTED);
  1176. assert(event_loop.active_timers.size() == 0);
  1177. // Now run through it again
  1178. base_process_service_test::handle_exit(&p, 0); // exit the daemon process
  1179. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  1180. assert(p.get_state() == service_state_t::STARTED);
  1181. assert(p.get_pid() == -1);
  1182. assert(daemon_instance == bp_sys::last_forked_pid);
  1183. event_loop.advance_time(time_val {0, 1000});
  1184. sset.process_queues();
  1185. // Now a new process should've been launched:
  1186. assert(daemon_instance + 1 == bp_sys::last_forked_pid);
  1187. assert(p.get_state() == service_state_t::STARTED);
  1188. assert(event_loop.active_timers.size() == 1);
  1189. base_process_service_test::exec_succeeded(&p);
  1190. sset.process_queues();
  1191. assert(event_loop.active_timers.size() == 1);
  1192. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1193. base_process_service_test::handle_exit(&p, 0);
  1194. sset.process_queues();
  1195. assert(p.get_state() == service_state_t::STARTED);
  1196. assert(p.get_pid() == daemon_instance);
  1197. assert(event_loop.active_timers.size() == 0);
  1198. sset.remove_service(&p);
  1199. }
  1200. // stop issued during smooth recovery, with dependent
  1201. void test_bgproc_smooth_recove2()
  1202. {
  1203. using namespace std;
  1204. service_set sset;
  1205. ha_string command = "test-command";
  1206. list<pair<unsigned,unsigned>> command_offsets;
  1207. command_offsets.emplace_back(0, command.length());
  1208. bgproc_service p {&sset, "testproc", ha_string(command), command_offsets, {}};
  1209. init_service_defaults(p);
  1210. p.set_smooth_recovery(true);
  1211. p.set_restart_delay(time_val {0, 1000});
  1212. p.set_pid_file("/run/daemon.pid");
  1213. sset.add_service(&p);
  1214. process_service d1 {&sset, "testproc-d1", std::move(command), command_offsets, {{&p, REG}}};
  1215. init_service_defaults(d1);
  1216. sset.add_service(&d1);
  1217. d1.start();
  1218. sset.process_queues();
  1219. // process for p exec succeds, reads pid file, starts
  1220. base_process_service_test::exec_succeeded(&p);
  1221. sset.process_queues();
  1222. pid_t daemon_instance;
  1223. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1224. assert(p.get_state() == service_state_t::STARTING);
  1225. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1226. sset.process_queues();
  1227. assert(p.get_state() == service_state_t::STARTED);
  1228. // dependent has been forked already:
  1229. assert(bp_sys::last_forked_pid == daemon_instance + 1);
  1230. // dependent then starts
  1231. base_process_service_test::exec_succeeded(&d1);
  1232. assert(bp_sys::last_forked_pid == daemon_instance + 1);
  1233. // exit daemon process unexpectedly:
  1234. base_process_service_test::handle_exit(&p, 0);
  1235. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  1236. assert(p.get_state() == service_state_t::STARTED);
  1237. assert(bp_sys::last_forked_pid == daemon_instance + 1);
  1238. event_loop.advance_time(time_val {0, 1000});
  1239. // Now a new process should've been launched:
  1240. assert(bp_sys::last_forked_pid == daemon_instance + 2);
  1241. assert(p.get_state() == service_state_t::STARTED);
  1242. assert(event_loop.active_timers.size() == 1); // start timer
  1243. // We tell the service to stop, before the smooth recovery completes (in fact
  1244. // before the exec even succeeds):
  1245. p.stop(true);
  1246. sset.process_queues();
  1247. assert(p.get_state() == service_state_t::STOPPING);
  1248. assert(d1.get_state() == service_state_t::STOPPING);
  1249. // Now the bgprocess launcher completes but with a bogus pid file contents:
  1250. string pid_file_content_str = "";
  1251. vector<char> pid_file_content_v(pid_file_content_str.begin(), pid_file_content_str.end());
  1252. bp_sys::supply_file_content("/run/daemon.pid", std::move(pid_file_content_v));
  1253. base_process_service_test::exec_succeeded(&p);
  1254. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1255. assert(p.get_state() == service_state_t::STOPPING);
  1256. assert(d1.get_state() == service_state_t::STOPPING);
  1257. // Now the dependent stops:
  1258. base_process_service_test::handle_exit(&d1, 0);
  1259. assert(d1.get_state() == service_state_t::STOPPED);
  1260. assert(p.get_state() == service_state_t::STOPPED);
  1261. assert(event_loop.active_timers.size() == 0);
  1262. sset.remove_service(&p);
  1263. sset.remove_service(&d1);
  1264. }
  1265. // bgproc smooth recovery failure
  1266. void test_bgproc_smooth_recove3()
  1267. {
  1268. using namespace std;
  1269. service_set sset;
  1270. ha_string command = "test-command";
  1271. list<pair<unsigned,unsigned>> command_offsets;
  1272. command_offsets.emplace_back(0, command.length());
  1273. bgproc_service p {&sset, "testproc", ha_string(command), command_offsets, {}};
  1274. init_service_defaults(p);
  1275. p.set_smooth_recovery(true);
  1276. p.set_restart_delay(time_val {0, 1000});
  1277. p.set_pid_file("/run/daemon.pid");
  1278. sset.add_service(&p);
  1279. p.start();
  1280. sset.process_queues();
  1281. // process for p exec succeds, reads pid file, starts
  1282. base_process_service_test::exec_succeeded(&p);
  1283. sset.process_queues();
  1284. pid_t daemon_instance;
  1285. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1286. assert(p.get_state() == service_state_t::STARTING);
  1287. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1288. sset.process_queues();
  1289. assert(p.get_state() == service_state_t::STARTED);
  1290. // exit daemon process unexpectedly:
  1291. base_process_service_test::handle_exit(&p, 0);
  1292. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  1293. assert(p.get_state() == service_state_t::STARTED);
  1294. assert(bp_sys::last_forked_pid == daemon_instance);
  1295. event_loop.advance_time(time_val {0, 1000});
  1296. // Now a new process should've been launched:
  1297. assert(bp_sys::last_forked_pid == daemon_instance + 1);
  1298. assert(p.get_state() == service_state_t::STARTED);
  1299. assert(event_loop.active_timers.size() == 1); // start timer
  1300. // Let the new process fail to executable:
  1301. base_process_service_test::exec_failed(&p, ENOMEM);
  1302. sset.process_queues();
  1303. assert(p.get_state() == service_state_t::STOPPED);
  1304. assert(p.get_stop_reason() == stopped_reason_t::TERMINATED);
  1305. assert(event_loop.active_timers.size() == 0);
  1306. sset.remove_service(&p);
  1307. }
  1308. // stop while in smooth recovery - waiting for restart timer
  1309. void test_bgproc_smooth_recove4()
  1310. {
  1311. using namespace std;
  1312. service_set sset;
  1313. ha_string command = "test-command";
  1314. list<pair<unsigned,unsigned>> command_offsets;
  1315. command_offsets.emplace_back(0, command.length());
  1316. bgproc_service p {&sset, "testproc", ha_string(command), command_offsets, {}};
  1317. init_service_defaults(p);
  1318. p.set_smooth_recovery(true);
  1319. p.set_restart_delay(time_val {0, 1000});
  1320. p.set_pid_file("/run/daemon.pid");
  1321. sset.add_service(&p);
  1322. p.start();
  1323. sset.process_queues();
  1324. // process for p exec succeds, reads pid file, starts
  1325. base_process_service_test::exec_succeeded(&p);
  1326. sset.process_queues();
  1327. pid_t daemon_instance;
  1328. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1329. assert(p.get_state() == service_state_t::STARTING);
  1330. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1331. sset.process_queues();
  1332. assert(p.get_state() == service_state_t::STARTED);
  1333. // exit daemon process unexpectedly:
  1334. base_process_service_test::handle_exit(&p, 0);
  1335. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  1336. assert(p.get_state() == service_state_t::STARTED);
  1337. assert(bp_sys::last_forked_pid == daemon_instance);
  1338. // Now issue stop:
  1339. p.stop();
  1340. sset.process_queues();
  1341. // since process was not running, shouldn't need to wait for it to end:
  1342. assert(p.get_state() == service_state_t::STOPPED);
  1343. assert(event_loop.active_timers.size() == 0);
  1344. assert(bp_sys::last_forked_pid == daemon_instance);
  1345. // now start again:
  1346. p.start();
  1347. sset.process_queues();
  1348. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1349. assert(p.get_state() == service_state_t::STARTING);
  1350. base_process_service_test::exec_succeeded(&p);
  1351. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1352. sset.process_queues();
  1353. assert(p.get_state() == service_state_t::STARTED);
  1354. // and terminate:
  1355. p.stop();
  1356. sset.process_queues();
  1357. base_process_service_test::handle_exit(&p, 0);
  1358. sset.process_queues();
  1359. assert(p.get_state() == service_state_t::STOPPED);
  1360. sset.remove_service(&p);
  1361. }
  1362. // Unexpected termination with restart
  1363. void test_bgproc_term_restart()
  1364. {
  1365. using namespace std;
  1366. service_set sset;
  1367. ha_string command = "test-command";
  1368. list<pair<unsigned,unsigned>> command_offsets;
  1369. command_offsets.emplace_back(0, command.length());
  1370. std::list<prelim_dep> depends;
  1371. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1372. init_service_defaults(p);
  1373. p.set_auto_restart(auto_restart_mode::ALWAYS);
  1374. p.set_restart_delay(time_val {0, 1000});
  1375. p.set_pid_file("/run/daemon.pid");
  1376. sset.add_service(&p);
  1377. p.start();
  1378. sset.process_queues();
  1379. base_process_service_test::exec_succeeded(&p);
  1380. sset.process_queues();
  1381. assert(p.get_state() == service_state_t::STARTING);
  1382. pid_t daemon_instance;
  1383. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1384. // Launch process completes:
  1385. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1386. assert(p.get_state() == service_state_t::STARTED);
  1387. assert(p.get_pid() == daemon_instance);
  1388. // Unexpected termination of daemon:
  1389. base_process_service_test::handle_exit(&p, 1);
  1390. // Should re-start:
  1391. assert(p.get_state() == service_state_t::STARTING);
  1392. // Starting, restart timer should be armed:
  1393. assert(p.get_state() == service_state_t::STARTING);
  1394. assert(p.get_pid() == -1);
  1395. assert(event_loop.active_timers.size() == 1);
  1396. event_loop.advance_time(time_val(0, 1000));
  1397. assert(event_loop.active_timers.size() == 1); // start timer
  1398. assert(p.get_pid() != -1);
  1399. // Supply new pid
  1400. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1401. // Launch process completes again
  1402. base_process_service_test::handle_exit(&p, 0);
  1403. assert(p.get_state() == service_state_t::STARTED);
  1404. assert(p.get_pid() == daemon_instance);
  1405. assert(event_loop.active_timers.size() == 0);
  1406. sset.remove_service(&p);
  1407. }
  1408. void test_bgproc_stop()
  1409. {
  1410. using namespace std;
  1411. service_set sset;
  1412. ha_string command = "test-command";
  1413. list<pair<unsigned,unsigned>> command_offsets;
  1414. command_offsets.emplace_back(0, command.length());
  1415. std::list<prelim_dep> depends;
  1416. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1417. init_service_defaults(p);
  1418. p.set_auto_restart(auto_restart_mode::ALWAYS);
  1419. p.set_pid_file("/run/daemon.pid");
  1420. sset.add_service(&p);
  1421. p.start();
  1422. sset.process_queues();
  1423. base_process_service_test::exec_succeeded(&p);
  1424. sset.process_queues();
  1425. assert(p.get_state() == service_state_t::STARTING);
  1426. pid_t daemon_instance;
  1427. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1428. // Launch process completes:
  1429. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1430. assert(p.get_state() == service_state_t::STARTED);
  1431. assert(p.get_pid() == daemon_instance);
  1432. // Issue stop:
  1433. p.stop();
  1434. sset.process_queues();
  1435. assert(p.get_state() == service_state_t::STOPPING);
  1436. base_process_service_test::handle_exit(&p, 1);
  1437. assert(p.get_state() == service_state_t::STOPPED);
  1438. assert(event_loop.active_timers.size() == 0);
  1439. sset.remove_service(&p);
  1440. }
  1441. // If launch process completes successfully (exit code 0) after "stop" issued during startup,
  1442. // make sure the pid file is read and the daemon process is signalled.
  1443. void test_bgproc_stop2()
  1444. {
  1445. using namespace std;
  1446. service_set sset;
  1447. ha_string command = "test-command";
  1448. list<pair<unsigned,unsigned>> command_offsets;
  1449. command_offsets.emplace_back(0, command.length());
  1450. std::list<prelim_dep> depends;
  1451. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1452. init_service_defaults(p);
  1453. p.set_auto_restart(auto_restart_mode::ALWAYS);
  1454. p.set_pid_file("/run/daemon.pid");
  1455. sset.add_service(&p);
  1456. p.start();
  1457. sset.process_queues();
  1458. base_process_service_test::exec_succeeded(&p);
  1459. sset.process_queues();
  1460. assert(p.get_state() == service_state_t::STARTING);
  1461. pid_t daemon_instance;
  1462. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1463. // Now issue stop
  1464. p.stop();
  1465. // doesn't really matter if state is STARTING or STOPPING
  1466. assert(p.get_state() == service_state_t::STARTING || p.get_state() == service_state_t::STOPPING);
  1467. // Launch process completes:
  1468. bp_sys::last_sig_sent = -1;
  1469. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1470. // What should happen now: read the pid file, immediately signal the daemon, and go STOPPING
  1471. assert(p.get_pid() == daemon_instance);
  1472. assert(bp_sys::last_sig_sent == SIGTERM);
  1473. assert(p.get_state() == service_state_t::STOPPING);
  1474. // daemon exits:
  1475. base_process_service_test::handle_exit(&p, 1);
  1476. assert(p.get_state() == service_state_t::STOPPED);
  1477. assert(event_loop.active_timers.size() == 0);
  1478. sset.remove_service(&p);
  1479. }
  1480. // Stop issued during smooth recovery
  1481. void test_bgproc_stop3()
  1482. {
  1483. using namespace std;
  1484. service_set sset;
  1485. ha_string command = "test-command";
  1486. list<pair<unsigned,unsigned>> command_offsets;
  1487. command_offsets.emplace_back(0, command.length());
  1488. std::list<prelim_dep> depends;
  1489. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1490. init_service_defaults(p);
  1491. p.set_smooth_recovery(true);
  1492. p.set_restart_delay(time_val {0, 1000});
  1493. p.set_pid_file("/run/daemon.pid");
  1494. sset.add_service(&p);
  1495. p.start();
  1496. sset.process_queues();
  1497. base_process_service_test::exec_succeeded(&p);
  1498. sset.process_queues();
  1499. pid_t daemon_instance;
  1500. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1501. assert(p.get_state() == service_state_t::STARTING);
  1502. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1503. sset.process_queues();
  1504. // daemon process has been started now, state should be STARTED
  1505. assert(p.get_state() == service_state_t::STARTED);
  1506. assert(daemon_instance == bp_sys::last_forked_pid);
  1507. base_process_service_test::handle_exit(&p, 0); // exit the daemon process
  1508. // since time hasn't been changed, we expect that the process has not yet been re-launched:
  1509. assert(p.get_state() == service_state_t::STARTED);
  1510. assert(daemon_instance == bp_sys::last_forked_pid);
  1511. event_loop.advance_time(time_val {0, 1000});
  1512. sset.process_queues();
  1513. // Now a new process should've been launched:
  1514. assert(daemon_instance + 1 == bp_sys::last_forked_pid);
  1515. assert(p.get_state() == service_state_t::STARTED);
  1516. assert(event_loop.active_timers.size() == 1); // start timer
  1517. base_process_service_test::exec_succeeded(&p);
  1518. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1519. // Now, we are in smooth recovery (state = STARTED), launch process is running.
  1520. p.stop();
  1521. assert(p.get_state() == service_state_t::STOPPING);
  1522. assert(event_loop.active_timers.size() == 1);
  1523. base_process_service_test::handle_exit(&p, 0);
  1524. // The launch process terminated, but *successfully*. So the PID file should be read and the
  1525. // daemon process killed also.
  1526. assert(p.get_state() == service_state_t::STOPPING);
  1527. assert(p.get_pid() == daemon_instance);
  1528. base_process_service_test::handle_exit(&p, 0); // exit the daemon process
  1529. assert(p.get_state() == service_state_t::STOPPED);
  1530. assert(daemon_instance == bp_sys::last_forked_pid);
  1531. assert(event_loop.active_timers.size() == 0);
  1532. sset.remove_service(&p);
  1533. }
  1534. // stop issued via command (service with stop-command set)
  1535. void test_bgproc_stop4()
  1536. {
  1537. using namespace std;
  1538. service_set sset;
  1539. ha_string command = "test-command";
  1540. list<pair<unsigned,unsigned>> command_offsets;
  1541. command_offsets.emplace_back(0, command.length());
  1542. std::list<prelim_dep> depends;
  1543. ha_string stop_command = "stop-command";
  1544. list<pair<unsigned,unsigned>> stop_command_offsets;
  1545. stop_command_offsets.emplace_back(0, stop_command.length());
  1546. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1547. init_service_defaults(p);
  1548. p.set_smooth_recovery(true);
  1549. p.set_restart_delay(time_val {0, 1000});
  1550. p.set_pid_file("/run/daemon.pid");
  1551. sset.add_service(&p);
  1552. p.set_stop_command(stop_command, stop_command_offsets);
  1553. p.start();
  1554. sset.process_queues();
  1555. base_process_service_test::exec_succeeded(&p);
  1556. sset.process_queues();
  1557. pid_t daemon_instance;
  1558. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1559. assert(p.get_state() == service_state_t::STARTING);
  1560. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1561. sset.process_queues();
  1562. // daemon process has been started now, state should be STARTED
  1563. assert(p.get_state() == service_state_t::STARTED);
  1564. assert(daemon_instance == bp_sys::last_forked_pid);
  1565. // so stop:
  1566. p.stop();
  1567. sset.process_queues();
  1568. base_process_service_test::handle_stop_exit(&p, 0); // exit the daemon process
  1569. sset.process_queues();
  1570. assert(p.get_state() == service_state_t::STOPPING);
  1571. base_process_service_test::handle_exit(&p, 0);
  1572. sset.process_queues();
  1573. assert(p.get_state() == service_state_t::STOPPED);
  1574. assert(event_loop.active_timers.size() == 0);
  1575. sset.remove_service(&p);
  1576. }
  1577. // stop issued via command (service with stop-command set); service process dies before stop command
  1578. void test_bgproc_stop5()
  1579. {
  1580. using namespace std;
  1581. service_set sset;
  1582. ha_string command = "test-command";
  1583. list<pair<unsigned,unsigned>> command_offsets;
  1584. command_offsets.emplace_back(0, command.length());
  1585. std::list<prelim_dep> depends;
  1586. ha_string stop_command = "stop-command";
  1587. list<pair<unsigned,unsigned>> stop_command_offsets;
  1588. stop_command_offsets.emplace_back(0, stop_command.length());
  1589. bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1590. init_service_defaults(p);
  1591. p.set_smooth_recovery(true);
  1592. p.set_restart_delay(time_val {0, 1000});
  1593. p.set_pid_file("/run/daemon.pid");
  1594. sset.add_service(&p);
  1595. p.set_stop_command(stop_command, stop_command_offsets);
  1596. p.start();
  1597. sset.process_queues();
  1598. base_process_service_test::exec_succeeded(&p);
  1599. sset.process_queues();
  1600. pid_t daemon_instance;
  1601. supply_pid_contents("/run/daemon.pid", &daemon_instance);
  1602. assert(p.get_state() == service_state_t::STARTING);
  1603. base_process_service_test::handle_exit(&p, 0); // exit the launch process
  1604. sset.process_queues();
  1605. // daemon process has been started now, state should be STARTED
  1606. assert(p.get_state() == service_state_t::STARTED);
  1607. assert(daemon_instance == bp_sys::last_forked_pid);
  1608. // so stop:
  1609. p.stop();
  1610. sset.process_queues();
  1611. assert(p.get_state() == service_state_t::STOPPING);
  1612. base_process_service_test::handle_exit(&p, 0);
  1613. sset.process_queues();
  1614. assert(p.get_state() == service_state_t::STOPPING);
  1615. base_process_service_test::handle_stop_exit(&p, 0); // exit the daemon process
  1616. sset.process_queues();
  1617. assert(p.get_state() == service_state_t::STOPPED);
  1618. assert(event_loop.active_timers.size() == 0);
  1619. sset.remove_service(&p);
  1620. }
  1621. // Test stop timeout
  1622. void test_scripted_stop_timeout()
  1623. {
  1624. using namespace std;
  1625. service_set sset;
  1626. ha_string command = "test-command";
  1627. ha_string stopcommand = "stop-command";
  1628. list<pair<unsigned,unsigned>> command_offsets;
  1629. command_offsets.emplace_back(0, command.length());
  1630. std::list<prelim_dep> depends;
  1631. scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
  1632. init_service_defaults(p);
  1633. p.set_stop_command(stopcommand, command_offsets);
  1634. p.set_stop_timeout(time_val {10, 0});
  1635. sset.add_service(&p);
  1636. p.start();
  1637. sset.process_queues();
  1638. assert(p.get_state() == service_state_t::STARTING);
  1639. base_process_service_test::exec_succeeded(&p);
  1640. sset.process_queues();
  1641. base_process_service_test::handle_exit(&p, 0);
  1642. sset.process_queues();
  1643. assert(p.get_state() == service_state_t::STARTED);
  1644. p.stop(true);
  1645. sset.process_queues();
  1646. assert(p.get_state() == service_state_t::STOPPING);
  1647. base_process_service_test::exec_succeeded(&p);
  1648. sset.process_queues();
  1649. // should still be stopping:
  1650. assert(p.get_state() == service_state_t::STOPPING);
  1651. event_loop.advance_time(time_val {10, 0}); // expire stop timer
  1652. sset.process_queues();
  1653. // kill signal (SIGKILL) should have been sent; process not dead until it's dead, however
  1654. assert(p.get_state() == service_state_t::STOPPING);
  1655. assert(bp_sys::last_sig_sent == SIGKILL);
  1656. base_process_service_test::handle_exit(&p, SIGKILL);
  1657. sset.process_queues();
  1658. assert(p.get_state() == service_state_t::STOPPED);
  1659. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  1660. assert(event_loop.active_timers.size() == 0);
  1661. sset.remove_service(&p);
  1662. }
  1663. void test_scripted_start_fail()
  1664. {
  1665. using namespace std;
  1666. service_set sset;
  1667. ha_string command = "test-command";
  1668. ha_string stopcommand = "stop-command";
  1669. list<pair<unsigned,unsigned>> command_offsets;
  1670. command_offsets.emplace_back(0, command.length());
  1671. std::list<prelim_dep> depends;
  1672. scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
  1673. init_service_defaults(p);
  1674. p.set_stop_command(stopcommand, command_offsets);
  1675. sset.add_service(&p);
  1676. service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}});
  1677. service_record *s3 = new service_record(&sset, "test-service-3",
  1678. service_type_t::INTERNAL, {{&p, REG}, {s2, REG}});
  1679. sset.add_service(s2);
  1680. sset.add_service(s3);
  1681. s3->start();
  1682. sset.process_queues();
  1683. assert(p.get_state() == service_state_t::STARTING);
  1684. base_process_service_test::exec_succeeded(&p);
  1685. sset.process_queues();
  1686. base_process_service_test::handle_exit(&p, 0x1); // exit fail
  1687. sset.process_queues();
  1688. // failed to start:
  1689. assert(p.get_state() == service_state_t::STOPPED);
  1690. assert(s2->get_state() == service_state_t::STOPPED);
  1691. assert(s3->get_state() == service_state_t::STOPPED);
  1692. assert(p.get_stop_reason() == stopped_reason_t::FAILED);
  1693. assert(s2->get_stop_reason() == stopped_reason_t::DEPFAILED);
  1694. assert(s3->get_stop_reason() == stopped_reason_t::DEPFAILED);
  1695. event_loop.active_timers.clear();
  1696. sset.remove_service(&p);
  1697. assert(sset.count_active_services() == 0);
  1698. }
  1699. void test_scripted_stop_fail()
  1700. {
  1701. using namespace std;
  1702. service_set sset;
  1703. ha_string command = "test-command";
  1704. ha_string stopcommand = "stop-command";
  1705. list<pair<unsigned,unsigned>> command_offsets;
  1706. command_offsets.emplace_back(0, command.length());
  1707. std::list<prelim_dep> depends;
  1708. scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
  1709. init_service_defaults(p);
  1710. p.set_stop_command(stopcommand, command_offsets);
  1711. sset.add_service(&p);
  1712. service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {});
  1713. service_record *s3 = new service_record(&sset, "test-service-3", service_type_t::INTERNAL,
  1714. {{s2, REG}, {&p, REG}});
  1715. service_record *s4 = new service_record(&sset, "test-service-4", service_type_t::INTERNAL,
  1716. {{&p, REG}, {s3, REG}});
  1717. sset.add_service(s2);
  1718. sset.add_service(s3);
  1719. sset.add_service(s4);
  1720. s4->start();
  1721. sset.process_queues();
  1722. base_process_service_test::exec_succeeded(&p);
  1723. sset.process_queues();
  1724. base_process_service_test::handle_exit(&p, 0x0); // success
  1725. sset.process_queues();
  1726. assert(p.get_state() == service_state_t::STARTED);
  1727. assert(s2->get_state() == service_state_t::STARTED);
  1728. assert(s3->get_state() == service_state_t::STARTED);
  1729. assert(s4->get_state() == service_state_t::STARTED);
  1730. pid_t last_forked = bp_sys::last_forked_pid;
  1731. s4->stop(true);
  1732. sset.process_queues();
  1733. base_process_service_test::exec_succeeded(&p);
  1734. sset.process_queues();
  1735. base_process_service_test::handle_exit(&p, 0x1); // failure
  1736. sset.process_queues();
  1737. // The stop command should be executed once:
  1738. assert((bp_sys::last_forked_pid - last_forked) == 1);
  1739. assert(p.get_state() == service_state_t::STOPPED);
  1740. assert(s2->get_state() == service_state_t::STOPPED);
  1741. assert(s3->get_state() == service_state_t::STOPPED);
  1742. assert(s4->get_state() == service_state_t::STOPPED);
  1743. event_loop.active_timers.clear();
  1744. sset.remove_service(&p);
  1745. }
  1746. void test_scripted_start_skip()
  1747. {
  1748. using namespace std;
  1749. service_set sset;
  1750. ha_string command = "test-command";
  1751. list<pair<unsigned,unsigned>> command_offsets;
  1752. command_offsets.emplace_back(0, command.length());
  1753. std::list<prelim_dep> depends;
  1754. scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
  1755. init_service_defaults(p);
  1756. service_flags_t sflags;
  1757. sflags.skippable = true;
  1758. p.set_flags(sflags);
  1759. sset.add_service(&p);
  1760. service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}});
  1761. sset.add_service(s2);
  1762. s2->start();
  1763. sset.process_queues();
  1764. assert(p.get_state() == service_state_t::STARTING);
  1765. base_process_service_test::exec_succeeded(&p);
  1766. sset.process_queues();
  1767. assert(p.get_state() == service_state_t::STARTING);
  1768. base_process_service_test::handle_signal_exit(&p, SIGINT); // interrupted
  1769. sset.process_queues();
  1770. assert(p.get_state() == service_state_t::STARTED);
  1771. assert(s2->get_state() == service_state_t::STARTED);
  1772. assert(p.was_start_skipped());
  1773. assert(! s2->was_start_skipped());
  1774. assert(sset.count_active_services() == 2);
  1775. s2->stop(true);
  1776. sset.process_queues();
  1777. assert(p.get_state() == service_state_t::STOPPED);
  1778. assert(s2->get_state() == service_state_t::STOPPED);
  1779. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  1780. assert(s2->get_stop_reason() == stopped_reason_t::NORMAL);
  1781. assert(sset.count_active_services() == 0);
  1782. event_loop.active_timers.clear();
  1783. sset.remove_service(&p);
  1784. }
  1785. // Test interrupting start of a service marked skippable
  1786. void test_scripted_start_skip2()
  1787. {
  1788. using namespace std;
  1789. service_set sset;
  1790. ha_string command = "test-command";
  1791. list<pair<unsigned,unsigned>> command_offsets;
  1792. command_offsets.emplace_back(0, command.length());
  1793. std::list<prelim_dep> depends;
  1794. scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
  1795. init_service_defaults(p);
  1796. service_flags_t sflags;
  1797. sflags.skippable = true;
  1798. sflags.start_interruptible = true;
  1799. p.set_flags(sflags);
  1800. sset.add_service(&p);
  1801. service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}});
  1802. sset.add_service(s2);
  1803. s2->start();
  1804. sset.process_queues();
  1805. assert(p.get_state() == service_state_t::STARTING);
  1806. base_process_service_test::exec_succeeded(&p);
  1807. sset.process_queues();
  1808. assert(p.get_state() == service_state_t::STARTING);
  1809. s2->stop(true); // abort startup; p should be cancelled
  1810. sset.process_queues();
  1811. assert(p.get_state() == service_state_t::STOPPING);
  1812. base_process_service_test::handle_signal_exit(&p, SIGINT); // interrupted
  1813. sset.process_queues();
  1814. assert(p.get_state() == service_state_t::STOPPED);
  1815. assert(s2->get_state() == service_state_t::STOPPED);
  1816. assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
  1817. assert(s2->get_stop_reason() == stopped_reason_t::NORMAL);
  1818. assert(sset.count_active_services() == 0);
  1819. event_loop.active_timers.clear();
  1820. sset.remove_service(&p);
  1821. }
  1822. // Test that starting a service with a waits-for dependency on another - currently stopping - service,
  1823. // causes that service to re-start.
  1824. void test_waitsfor_restart()
  1825. {
  1826. using namespace std;
  1827. service_set sset;
  1828. ha_string command = "test-command";
  1829. list<pair<unsigned,unsigned>> command_offsets;
  1830. command_offsets.emplace_back(0, command.length());
  1831. std::list<prelim_dep> depends;
  1832. process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  1833. init_service_defaults(p);
  1834. sset.add_service(&p);
  1835. service_record tp {&sset, "test-service", service_type_t::INTERNAL, {{&p, WAITS}}};
  1836. sset.add_service(&tp);
  1837. // start p:
  1838. p.start();
  1839. sset.process_queues();
  1840. assert(p.get_state() == service_state_t::STARTING);
  1841. base_process_service_test::exec_succeeded(&p);
  1842. sset.process_queues();
  1843. assert(p.get_state() == service_state_t::STARTED);
  1844. assert(event_loop.active_timers.size() == 0);
  1845. // begin stopping p:
  1846. p.stop(true);
  1847. sset.process_queues();
  1848. assert(p.get_state() == service_state_t::STOPPING);
  1849. // start tp (which waits-for p):
  1850. tp.start();
  1851. sset.process_queues();
  1852. assert(tp.get_state() == service_state_t::STARTING);
  1853. assert(p.get_state() == service_state_t::STOPPING);
  1854. // p terminates (finishes stopping). Then it should re-start...
  1855. base_process_service_test::handle_signal_exit(&p, SIGTERM);
  1856. sset.process_queues();
  1857. assert(tp.get_state() == service_state_t::STARTING);
  1858. assert(p.get_state() == service_state_t::STARTING);
  1859. base_process_service_test::exec_succeeded(&p);
  1860. sset.process_queues();
  1861. assert(tp.get_state() == service_state_t::STARTED);
  1862. assert(p.get_state() == service_state_t::STARTED);
  1863. sset.remove_service(&tp);
  1864. sset.remove_service(&p);
  1865. }
  1866. #define RUN_TEST(name, spacing) \
  1867. std::cout << #name "..." spacing << std::flush; \
  1868. name(); \
  1869. std::cout << "PASSED" << std::endl;
  1870. int main(int argc, char **argv)
  1871. {
  1872. RUN_TEST(test_proc_service_start, " ");
  1873. RUN_TEST(test_proc_notify_start, " ");
  1874. RUN_TEST(test_proc_unexpected_term, " ");
  1875. RUN_TEST(test_proc_term_start, " ");
  1876. RUN_TEST(test_proc_term_restart, " ");
  1877. RUN_TEST(test_proc_term_restart2, " ");
  1878. RUN_TEST(test_proc_term_restart3, " ");
  1879. RUN_TEST(test_proc_term_restart4, " ");
  1880. RUN_TEST(test_proc_term_restart5, " ");
  1881. RUN_TEST(test_proc_term_restart6, " ");
  1882. RUN_TEST(test_proc_term_restart_fail, "");
  1883. RUN_TEST(test_term_via_stop, " ");
  1884. RUN_TEST(test_term_via_stop2, " ");
  1885. RUN_TEST(test_term_via_stop3, " ");
  1886. RUN_TEST(test_proc_start_timeout, " ");
  1887. RUN_TEST(test_proc_start_timeout2, " ");
  1888. RUN_TEST(test_proc_start_execfail, " ");
  1889. RUN_TEST(test_proc_notify_fail, " ");
  1890. RUN_TEST(test_proc_stop_timeout, " ");
  1891. RUN_TEST(test_proc_smooth_recovery1, " ");
  1892. RUN_TEST(test_proc_smooth_recovery2, " ");
  1893. RUN_TEST(test_proc_smooth_recovery3, " ");
  1894. RUN_TEST(test_proc_smooth_recovery4, " ");
  1895. RUN_TEST(test_proc_smooth_recovery5, " ");
  1896. RUN_TEST(test_proc_smooth_recovery6, " ");
  1897. RUN_TEST(test_proc_smooth_recovery6a, "");
  1898. RUN_TEST(test_bgproc_start, " ");
  1899. RUN_TEST(test_bgproc_start_fail, " ");
  1900. RUN_TEST(test_bgproc_start_fail_pid, " ");
  1901. RUN_TEST(test_bgproc_unexpected_term, "");
  1902. RUN_TEST(test_bgproc_smooth_recover, " ");
  1903. RUN_TEST(test_bgproc_smooth_recove2, " ");
  1904. RUN_TEST(test_bgproc_smooth_recove3, " ");
  1905. RUN_TEST(test_bgproc_smooth_recove4, " ");
  1906. RUN_TEST(test_bgproc_term_restart, " ");
  1907. RUN_TEST(test_bgproc_stop, " ");
  1908. RUN_TEST(test_bgproc_stop2, " ");
  1909. RUN_TEST(test_bgproc_stop3, " ");
  1910. RUN_TEST(test_bgproc_stop4, " ");
  1911. RUN_TEST(test_bgproc_stop5, " ");
  1912. RUN_TEST(test_scripted_stop_timeout, " ");
  1913. RUN_TEST(test_scripted_start_fail, " ");
  1914. RUN_TEST(test_scripted_stop_fail, " ");
  1915. RUN_TEST(test_scripted_start_skip, " ");
  1916. RUN_TEST(test_scripted_start_skip2, " ");
  1917. RUN_TEST(test_waitsfor_restart, " ");
  1918. }