runtests.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592
  1. #!/usr/bin/env python3
  2. from __future__ import print_function, unicode_literals
  3. import argparse
  4. import coverage
  5. import json
  6. import logging
  7. import multiprocessing
  8. import os
  9. import shutil
  10. import subprocess
  11. import sys
  12. import threading
  13. import time
  14. RUNNER_PY = "nosetests"
  15. RUNNER_PY2 = "nosetests-2"
  16. RUNNER_PY3 = "nosetests-3"
  17. COVER_PY = "coverage"
  18. COVER_PY2 = "coverage2"
  19. COVER_PY3 = "coverage3"
  20. LASTLEN = None
  21. NUMREMAINING = None
  22. PRINTLOCK = None
  23. RUNNING = []
  24. FAILED = []
  25. NUMPROCS = multiprocessing.cpu_count() - 1
  26. if os.environ.get('BUILD_ID'):
  27. NUMPROCS = multiprocessing.cpu_count()
  28. def setup_parser():
  29. """ Set up the command line arguments supported and return the arguments
  30. """
  31. parser = argparse.ArgumentParser(description="Run the Pagure tests")
  32. parser.add_argument(
  33. "--debug",
  34. dest="debug",
  35. action="store_true",
  36. default=False,
  37. help="Increase the level of data logged.",
  38. )
  39. subparsers = parser.add_subparsers(title="actions")
  40. # RUN
  41. parser_run = subparsers.add_parser("run", help="Run the tests")
  42. parser_run.add_argument(
  43. "--py2",
  44. dest="py2",
  45. action="store_true",
  46. default=False,
  47. help="Runs the tests only in python2 instead of both python2 and python3",
  48. )
  49. parser_run.add_argument(
  50. "--py3",
  51. dest="py3",
  52. action="store_true",
  53. default=False,
  54. help="Runs the tests only in python3 instead of both python2 and python3",
  55. )
  56. parser_run.add_argument(
  57. "--results",
  58. default="results",
  59. help="Specify a folder in which the results should be placed "
  60. "(defaults to `results`)",
  61. )
  62. parser_run.add_argument(
  63. "-f",
  64. "--force",
  65. default=False,
  66. action="store_true",
  67. help="Override the results and newfailed file without asking you",
  68. )
  69. parser_run.add_argument(
  70. "--with-coverage",
  71. default=False,
  72. action="store_true",
  73. help="Also build coverage report",
  74. )
  75. parser_run.add_argument(
  76. "failed_tests",
  77. nargs="?",
  78. help="File containing a JSON list of the failed tests to run or "
  79. "pointing to a test file to run.",
  80. )
  81. parser_run.set_defaults(func=do_run)
  82. # RERUN
  83. parser_run = subparsers.add_parser("rerun", help="Run failed tests")
  84. parser_run.add_argument(
  85. "--debug",
  86. dest="debug",
  87. action="store_true",
  88. default=False,
  89. help="Expand the level of data returned.",
  90. )
  91. parser_run.add_argument(
  92. "--py2",
  93. dest="py2",
  94. action="store_true",
  95. default=False,
  96. help="Runs the tests only in python2 instead of both python2 and python3",
  97. )
  98. parser_run.add_argument(
  99. "--py3",
  100. dest="py3",
  101. action="store_true",
  102. default=False,
  103. help="Runs the tests only in python3 instead of both python2 and python3",
  104. )
  105. parser_run.add_argument(
  106. "--results",
  107. default="results",
  108. help="Specify a folder in which the results should be placed "
  109. "(defaults to `results`)",
  110. )
  111. parser_run.add_argument(
  112. "--with-coverage",
  113. default=False,
  114. action="store_true",
  115. help="Also build coverage report",
  116. )
  117. parser_run.set_defaults(func=do_rerun)
  118. # LIST
  119. parser_run = subparsers.add_parser("list", help="List failed tests")
  120. parser_run.add_argument(
  121. "--results",
  122. default="results",
  123. help="Specify a folder in which the results should be placed "
  124. "(defaults to `results`)",
  125. )
  126. parser_run.add_argument(
  127. "--show",
  128. default=False,
  129. action="store_true",
  130. help="Show the error files using `less`",
  131. )
  132. parser_run.add_argument(
  133. "-n", default=None, nargs="?", type=int,
  134. help="Number of failed test to show",
  135. )
  136. parser_run.set_defaults(func=do_list)
  137. # SHOW-COVERAGE
  138. parser_run = subparsers.add_parser(
  139. "show-coverage",
  140. help="Shows the coverage report from the data in the results folder")
  141. parser_run.add_argument(
  142. "--debug",
  143. dest="debug",
  144. action="store_true",
  145. default=False,
  146. help="Expand the level of data returned.",
  147. )
  148. parser_run.add_argument(
  149. "--py2",
  150. dest="py2",
  151. action="store_true",
  152. default=False,
  153. help="Runs the tests only in python2 instead of both python2 and python3",
  154. )
  155. parser_run.add_argument(
  156. "--py3",
  157. dest="py3",
  158. action="store_true",
  159. default=False,
  160. help="Runs the tests only in python3 instead of both python2 and python3",
  161. )
  162. parser_run.add_argument(
  163. "--results",
  164. default="results",
  165. help="Specify a folder in which the results should be placed "
  166. "(defaults to `results`)",
  167. )
  168. parser_run.set_defaults(func=do_show_coverage)
  169. return parser
  170. def clean_line():
  171. global LASTLEN
  172. with PRINTLOCK:
  173. if LASTLEN is not None:
  174. print(" " * LASTLEN, end="\r")
  175. LASTLEN = None
  176. def print_running():
  177. global LASTLEN
  178. with PRINTLOCK:
  179. msg = "Running %d suites: %d remaining, %d failed" % (
  180. len(RUNNING),
  181. NUMREMAINING,
  182. len(FAILED),
  183. )
  184. LASTLEN = len(msg)
  185. print(msg, end="\r")
  186. def add_running(suite):
  187. global NUMREMAINING
  188. with PRINTLOCK:
  189. NUMREMAINING -= 1
  190. RUNNING.append(suite)
  191. clean_line()
  192. print_running()
  193. def remove_running(suite, failed):
  194. with PRINTLOCK:
  195. RUNNING.remove(suite)
  196. clean_line()
  197. status = 'passed'
  198. if failed:
  199. status = 'FAILED'
  200. print("Test suite %s: %s" % (status, suite))
  201. print_running()
  202. class WorkerThread(threading.Thread):
  203. def __init__(self, sem, pyver, suite, results, with_cover):
  204. name = "py%s-%s" % (pyver, suite)
  205. super(WorkerThread, self).__init__(name="worker-%s" % name)
  206. self.name = name
  207. self.sem = sem
  208. self.pyver = pyver
  209. self.suite = suite
  210. self.failed = None
  211. self.results = results
  212. self.with_cover = with_cover
  213. def run(self):
  214. with self.sem:
  215. add_running(self.name)
  216. with open(os.path.join(self.results, self.name), "w") as resfile:
  217. if self.pyver == 2:
  218. runner = RUNNER_PY2
  219. elif self.pyver == 3:
  220. runner = RUNNER_PY3
  221. else:
  222. runner = RUNNER_PY
  223. cmd = [runner, "-v", "tests.%s" % self.suite]
  224. if self.with_cover:
  225. cmd.append("--with-cover")
  226. env = os.environ.copy()
  227. env.update({
  228. "PAGURE_CONFIG": "../tests/test_config",
  229. "COVERAGE_FILE": os.path.join(
  230. self.results, "%s.coverage" % self.name
  231. ),
  232. "LANG": "en_US.UTF-8",
  233. })
  234. proc = subprocess.Popen(
  235. cmd, cwd=".", stdout=resfile, stderr=subprocess.STDOUT, env=env
  236. )
  237. res = proc.wait()
  238. if res == 0:
  239. self.failed = False
  240. else:
  241. self.failed = True
  242. if not self.failed is not True:
  243. with PRINTLOCK:
  244. FAILED.append(self.name)
  245. remove_running(self.name, self.failed)
  246. def do_run(args):
  247. """ Performs some checks and runs the tests.
  248. """
  249. # Some pre-flight checks
  250. if not os.path.exists("./.git") or not os.path.exists("./nosetests3"):
  251. print("Please run from a single level into the Pagure codebase")
  252. return 1
  253. if os.path.exists(args.results):
  254. if not args.force:
  255. print(
  256. "Results folder exists, please remove it so we do not clobber"
  257. " or use --force"
  258. )
  259. return 1
  260. else:
  261. shutil.rmtree(args.results)
  262. os.mkdir(args.results)
  263. print("Pre-flight checks passed")
  264. suites = []
  265. if args.failed_tests:
  266. here = os.path.join(os.path.dirname(os.path.abspath(__file__)))
  267. failed_tests_fullpath = os.path.join(here, args.failed_tests)
  268. if not os.path.exists(failed_tests_fullpath):
  269. print("Could not find the specified file:%s" % failed_tests_fullpath)
  270. return 1
  271. print("Loading failed tests")
  272. try:
  273. with open(failed_tests_fullpath, "r") as ffile:
  274. suites = json.loads(ffile.read())
  275. except json.decoder.JSONDecodeError:
  276. bname = os.path.basename(args.failed_tests)
  277. if bname.endswith(".py") and bname.startswith("test_"):
  278. suites.append(bname.replace(".py", ""))
  279. if len(suites) == 0:
  280. print("Loading all tests")
  281. for fname in os.listdir("./tests"):
  282. if not fname.endswith(".py"):
  283. continue
  284. if not fname.startswith("test_"):
  285. continue
  286. suites.append(fname.replace(".py", ""))
  287. return _run_test_suites(args, suites)
  288. def do_rerun(args):
  289. """ Re-run tests that failed the last/specified run.
  290. """
  291. # Some pre-flight checks
  292. if not os.path.exists("./.git") or not os.path.exists("./pagure"):
  293. print("Please run from a single level into the Pagure codebase")
  294. return 1
  295. if not os.path.exists(args.results):
  296. print("Could not find an existing results folder at: %s" % args.results)
  297. return 1
  298. if not os.path.exists(os.path.join(args.results, "newfailed")):
  299. print(
  300. "Could not find an failed tests in the results folder at: %s" % args.results
  301. )
  302. return 1
  303. print("Pre-flight checks passed")
  304. suites = []
  305. tmp = []
  306. print("Loading failed tests")
  307. try:
  308. with open(os.path.join(args.results, "newfailed"), "r") as ffile:
  309. tmp = json.loads(ffile.read())
  310. except json.decoder.JSONDecodeError:
  311. print("File containing the failed tests is not JSON")
  312. return 1
  313. for suite in tmp:
  314. if suite.startswith(("py2-", "py3-")):
  315. suites.append(suite[4:])
  316. return _run_test_suites(args, set(suites))
  317. def _get_pyvers(args):
  318. pyvers = [2, 3]
  319. if args.py2:
  320. pyvers = [2,]
  321. elif args.py3:
  322. pyvers = [3,]
  323. un_versioned = False
  324. try:
  325. subprocess.check_call(["which", RUNNER_PY])
  326. un_versioned = True
  327. except subprocess.CalledProcessError:
  328. print("No %s found no unversioned runner" % RUNNER_PY)
  329. if 2 in pyvers:
  330. nopy2 = False
  331. try:
  332. subprocess.check_call(["which", RUNNER_PY2])
  333. except subprocess.CalledProcessError:
  334. print("No %s found, removing python 2" % RUNNER_PY2)
  335. del pyvers[pyvers.index(2)]
  336. if 3 in pyvers:
  337. nopy3 = False
  338. try:
  339. subprocess.check_call(["which", RUNNER_PY3])
  340. except subprocess.CalledProcessError:
  341. print("No %s found, removing python 3" % RUNNER_PY3)
  342. del pyvers[pyvers.index(3)]
  343. if not pyvers and un_versioned:
  344. pyvers = [""]
  345. return pyvers
  346. def _run_test_suites(args, suites):
  347. print("Using %d processes" % NUMPROCS)
  348. print("Start timing")
  349. start = time.time()
  350. global PRINTLOCK
  351. PRINTLOCK = threading.RLock()
  352. global NUMREMAINING
  353. NUMREMAINING = 0
  354. sem = threading.BoundedSemaphore(NUMPROCS)
  355. # Create a worker per test
  356. workers = {}
  357. pyvers = _get_pyvers(args)
  358. if not pyvers:
  359. return 1
  360. for suite in suites:
  361. for pyver in pyvers:
  362. NUMREMAINING += 1
  363. workers["py%s-%s" % (pyver, suite)] = WorkerThread(
  364. sem, pyver, suite, args.results, args.with_coverage
  365. )
  366. # Start the workers
  367. print("Starting the workers")
  368. print()
  369. print()
  370. for worker in workers.values():
  371. worker.start()
  372. # Wait for them to terminate
  373. for worker in workers:
  374. workers[worker].join()
  375. print_running()
  376. print()
  377. print("All work done")
  378. # Gather results
  379. print()
  380. print()
  381. print("Failed tests:")
  382. for worker in workers:
  383. if not workers[worker].failed:
  384. continue
  385. print("FAILED test: %s" % (worker))
  386. # Write failed
  387. if FAILED:
  388. with open(os.path.join(args.results, "newfailed"), "w") as ffile:
  389. ffile.write(json.dumps(FAILED))
  390. # Stats
  391. end = time.time()
  392. print()
  393. print()
  394. print(
  395. "Ran %d tests in %f seconds, of which %d failed"
  396. % (len(workers), (end - start), len(FAILED))
  397. )
  398. # Exit
  399. if len(FAILED) == 0:
  400. print("ALL PASSED! CONGRATULATIONS!")
  401. else:
  402. return 1
  403. if args.with_coverage:
  404. do_show_coverage(args)
  405. return 0
  406. def do_list(args):
  407. """ List tests that failed the last/specified run.
  408. """
  409. # Some pre-flight checks
  410. if not os.path.exists("./.git") or not os.path.exists("./pagure"):
  411. print("Please run from a single level into the Pagure codebase")
  412. return 1
  413. if not os.path.exists(args.results):
  414. print("Could not find an existing results folder at: %s" % args.results)
  415. return 1
  416. if not os.path.exists(os.path.join(args.results, "newfailed")):
  417. print(
  418. "Could not find an failed tests in the results folder at: %s" % args.results
  419. )
  420. return 1
  421. print("Pre-flight checks passed")
  422. suites = []
  423. tmp = []
  424. print("Loading failed tests")
  425. try:
  426. with open(os.path.join(args.results, "newfailed"), "r") as ffile:
  427. suites = json.loads(ffile.read())
  428. except json.decoder.JSONDecodeError:
  429. print("File containing the failed tests is not JSON")
  430. return 1
  431. print("Failed tests")
  432. failed_tests = len(suites)
  433. if args.n:
  434. suites = suites[:args.n]
  435. print("- " + "\n- ".join(suites))
  436. print("Total: %s test failed" % failed_tests)
  437. if args.show:
  438. for suite in suites:
  439. cmd = ["less", os.path.join(args.results, suite)]
  440. subprocess.check_call(cmd)
  441. def do_show_coverage(args):
  442. print()
  443. print("Combining coverage results...")
  444. pyvers = _get_pyvers(args)
  445. for pyver in pyvers:
  446. coverfiles = []
  447. for fname in os.listdir(args.results):
  448. if fname.endswith(".coverage") and fname.startswith("py%s-" % pyver):
  449. coverfiles.append(os.path.join(args.results, fname))
  450. cover = None
  451. if pyver == 2:
  452. cover = COVER_PY2
  453. elif pyver == 3:
  454. cover = COVER_PY3
  455. else:
  456. cover = COVER_PY
  457. env = {"COVERAGE_FILE": os.path.join(args.results, "combined.coverage")}
  458. cmd = [cover, "combine"] + coverfiles
  459. subprocess.check_call(cmd, env=env)
  460. print()
  461. print("Python %s coverage: " % pyver)
  462. cmd = [cover, "report", "--include=./pagure/*"]
  463. subprocess.check_call(cmd, env=env)
  464. def main():
  465. """ Main function """
  466. # Set up parser for global args
  467. parser = setup_parser()
  468. # Parse the commandline
  469. try:
  470. arg = parser.parse_args()
  471. except argparse.ArgumentTypeError as err:
  472. print("\nError: {0}".format(err))
  473. return 2
  474. logging.basicConfig()
  475. if arg.debug:
  476. LOG.setLevel(logging.DEBUG)
  477. if "func" not in arg:
  478. parser.print_help()
  479. return 1
  480. arg.results = os.path.abspath(arg.results)
  481. return_code = 0
  482. try:
  483. return_code = arg.func(arg)
  484. except KeyboardInterrupt:
  485. print("\nInterrupted by user.")
  486. return_code = 1
  487. except Exception as err:
  488. print("Error: {0}".format(err))
  489. logging.exception("Generic error caught:")
  490. return_code = 5
  491. return return_code
  492. if __name__ == "__main__":
  493. sys.exit(main())