1
0

runtests.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632
  1. #!/usr/bin/env python3
  2. from __future__ import print_function, unicode_literals, absolute_import
  3. import argparse
  4. import coverage
  5. import json
  6. import logging
  7. import multiprocessing
  8. import os
  9. import shutil
  10. import subprocess
  11. import sys
  12. import threading
  13. import time
  14. RUNNER_PY = "nosetests"
  15. RUNNER_PY2 = "nosetests-2"
  16. RUNNER_PY3 = "nosetests-3"
  17. COVER_PY = "coverage"
  18. COVER_PY2 = "coverage2"
  19. COVER_PY3 = "coverage3"
  20. LASTLEN = None
  21. NUMREMAINING = None
  22. PRINTLOCK = None
  23. RUNNING = []
  24. FAILED = []
  25. NUMPROCS = multiprocessing.cpu_count() - 1
  26. if os.environ.get('BUILD_ID'):
  27. NUMPROCS = multiprocessing.cpu_count()
  28. HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)))
  29. LOG = logging.getLogger(__name__)
  30. def setup_parser():
  31. """ Set up the command line arguments supported and return the arguments
  32. """
  33. parser = argparse.ArgumentParser(description="Run the Pagure tests")
  34. parser.add_argument(
  35. "--debug",
  36. dest="debug",
  37. action="store_true",
  38. default=False,
  39. help="Increase the level of data logged.",
  40. )
  41. subparsers = parser.add_subparsers(title="actions")
  42. # RUN
  43. parser_run = subparsers.add_parser("run", help="Run the tests")
  44. parser_run.add_argument(
  45. "--py2",
  46. dest="py2",
  47. action="store_true",
  48. default=False,
  49. help="Runs the tests only in python2 instead of both python2 and python3",
  50. )
  51. parser_run.add_argument(
  52. "--py3",
  53. dest="py3",
  54. action="store_true",
  55. default=False,
  56. help="Runs the tests only in python3 instead of both python2 and python3",
  57. )
  58. parser_run.add_argument(
  59. "--results",
  60. default="results",
  61. help="Specify a folder in which the results should be placed "
  62. "(defaults to `results`)",
  63. )
  64. parser_run.add_argument(
  65. "-f",
  66. "--force",
  67. default=False,
  68. action="store_true",
  69. help="Override the results and newfailed file without asking you",
  70. )
  71. parser_run.add_argument(
  72. "--with-coverage",
  73. default=False,
  74. action="store_true",
  75. help="Also build coverage report",
  76. )
  77. parser_run.add_argument(
  78. "failed_tests",
  79. nargs="?",
  80. help="File containing a JSON list of the failed tests to run or "
  81. "pointing to a test file to run.",
  82. )
  83. parser_run.set_defaults(func=do_run)
  84. # RERUN
  85. parser_run = subparsers.add_parser("rerun", help="Run failed tests")
  86. parser_run.add_argument(
  87. "--debug",
  88. dest="debug",
  89. action="store_true",
  90. default=False,
  91. help="Expand the level of data returned.",
  92. )
  93. parser_run.add_argument(
  94. "--py2",
  95. dest="py2",
  96. action="store_true",
  97. default=False,
  98. help="Runs the tests only in python2 instead of both python2 and python3",
  99. )
  100. parser_run.add_argument(
  101. "--py3",
  102. dest="py3",
  103. action="store_true",
  104. default=False,
  105. help="Runs the tests only in python3 instead of both python2 and python3",
  106. )
  107. parser_run.add_argument(
  108. "--results",
  109. default="results",
  110. help="Specify a folder in which the results should be placed "
  111. "(defaults to `results`)",
  112. )
  113. parser_run.add_argument(
  114. "--with-coverage",
  115. default=False,
  116. action="store_true",
  117. help="Also build coverage report",
  118. )
  119. parser_run.set_defaults(func=do_rerun)
  120. # LIST
  121. parser_run = subparsers.add_parser("list", help="List failed tests")
  122. parser_run.add_argument(
  123. "--results",
  124. default="results",
  125. help="Specify a folder in which the results should be placed "
  126. "(defaults to `results`)",
  127. )
  128. parser_run.add_argument(
  129. "--show",
  130. default=False,
  131. action="store_true",
  132. help="Show the error files using `less`",
  133. )
  134. parser_run.add_argument(
  135. "-n", default=None, nargs="?", type=int,
  136. help="Number of failed test to show",
  137. )
  138. parser_run.set_defaults(func=do_list)
  139. # SHOW-COVERAGE
  140. parser_run = subparsers.add_parser(
  141. "show-coverage",
  142. help="Shows the coverage report from the data in the results folder")
  143. parser_run.add_argument(
  144. "--debug",
  145. dest="debug",
  146. action="store_true",
  147. default=False,
  148. help="Expand the level of data returned.",
  149. )
  150. parser_run.add_argument(
  151. "--py2",
  152. dest="py2",
  153. action="store_true",
  154. default=False,
  155. help="Runs the tests only in python2 instead of both python2 and python3",
  156. )
  157. parser_run.add_argument(
  158. "--py3",
  159. dest="py3",
  160. action="store_true",
  161. default=False,
  162. help="Runs the tests only in python3 instead of both python2 and python3",
  163. )
  164. parser_run.add_argument(
  165. "--results",
  166. default="results",
  167. help="Specify a folder in which the results should be placed "
  168. "(defaults to `results`)",
  169. )
  170. parser_run.set_defaults(func=do_show_coverage)
  171. return parser
  172. def clean_line():
  173. global LASTLEN
  174. with PRINTLOCK:
  175. if LASTLEN is not None:
  176. print(" " * LASTLEN, end="\r")
  177. LASTLEN = None
  178. def print_running():
  179. global LASTLEN
  180. with PRINTLOCK:
  181. msg = "Running %d suites: %d remaining, %d failed" % (
  182. len(RUNNING),
  183. NUMREMAINING,
  184. len(FAILED),
  185. )
  186. LASTLEN = len(msg)
  187. print(msg, end="\r")
  188. def add_running(suite):
  189. global NUMREMAINING
  190. with PRINTLOCK:
  191. NUMREMAINING -= 1
  192. RUNNING.append(suite)
  193. clean_line()
  194. print_running()
  195. def remove_running(suite, failed):
  196. with PRINTLOCK:
  197. RUNNING.remove(suite)
  198. clean_line()
  199. status = 'passed'
  200. if failed:
  201. status = 'FAILED'
  202. print("Test suite %s: %s" % (status, suite))
  203. print_running()
  204. class WorkerThread(threading.Thread):
  205. def __init__(self, sem, pyver, suite, results, with_cover):
  206. name = "py%s-%s" % (pyver, suite)
  207. super(WorkerThread, self).__init__(name="worker-%s" % name)
  208. self.name = name
  209. self.sem = sem
  210. self.pyver = pyver
  211. self.suite = suite
  212. self.failed = None
  213. self.results = results
  214. self.with_cover = with_cover
  215. def run(self):
  216. with self.sem:
  217. add_running(self.name)
  218. with open(os.path.join(self.results, self.name), "w") as resfile:
  219. if self.pyver == 2:
  220. runner = RUNNER_PY2
  221. elif self.pyver == 3:
  222. runner = RUNNER_PY3
  223. else:
  224. runner = RUNNER_PY
  225. cmd = [runner, "-v", "tests.%s" % self.suite]
  226. if self.with_cover:
  227. cmd.append("--with-cover")
  228. env = os.environ.copy()
  229. env.update({
  230. "PAGURE_CONFIG": "../tests/test_config",
  231. "COVERAGE_FILE": os.path.join(
  232. self.results, "%s.coverage" % self.name
  233. ),
  234. "LANG": "en_US.UTF-8",
  235. })
  236. proc = subprocess.Popen(
  237. cmd, cwd=".", stdout=resfile, stderr=subprocess.STDOUT, env=env
  238. )
  239. res = proc.wait()
  240. if res == 0:
  241. self.failed = False
  242. else:
  243. self.failed = True
  244. if not self.failed is not True:
  245. with PRINTLOCK:
  246. FAILED.append(self.name)
  247. remove_running(self.name, self.failed)
  248. def do_run(args):
  249. """ Performs some checks and runs the tests.
  250. """
  251. # Some pre-flight checks
  252. if not os.path.exists("./.git") or not os.path.exists("./nosetests3"):
  253. print("Please run from a single level into the Pagure codebase")
  254. return 1
  255. if os.path.exists(args.results):
  256. if not args.force:
  257. print(
  258. "Results folder exists, please remove it so we do not clobber"
  259. " or use --force"
  260. )
  261. return 1
  262. else:
  263. try:
  264. shutil.rmtree(args.results)
  265. os.mkdir(args.results)
  266. except:
  267. print(
  268. "Could not delete the %s directory, it will be "
  269. "wiped clean" % args.results)
  270. for content in os.listdir(args.results):
  271. os.remove(content)
  272. else:
  273. os.mkdir(args.results)
  274. print("Pre-flight checks passed")
  275. suites = []
  276. if args.failed_tests:
  277. here = os.path.join(os.path.dirname(os.path.abspath(__file__)))
  278. failed_tests_fullpath = os.path.join(here, args.failed_tests)
  279. if not os.path.exists(failed_tests_fullpath):
  280. print("Could not find the specified file:%s" % failed_tests_fullpath)
  281. return 1
  282. print("Loading failed tests")
  283. try:
  284. with open(failed_tests_fullpath, "r") as ffile:
  285. suites = json.loads(ffile.read())
  286. except:
  287. bname = os.path.basename(args.failed_tests)
  288. if bname.endswith(".py") and bname.startswith("test_"):
  289. suites.append(bname.replace(".py", ""))
  290. if len(suites) == 0:
  291. print("Loading all tests")
  292. for fname in os.listdir("./tests"):
  293. if not fname.endswith(".py"):
  294. continue
  295. if not fname.startswith("test_"):
  296. continue
  297. suites.append(fname.replace(".py", ""))
  298. return _run_test_suites(args, suites)
  299. def do_rerun(args):
  300. """ Re-run tests that failed the last/specified run.
  301. """
  302. # Some pre-flight checks
  303. if not os.path.exists("./.git") or not os.path.exists("./pagure"):
  304. print("Please run from a single level into the Pagure codebase")
  305. return 1
  306. if not os.path.exists(args.results):
  307. print("Could not find an existing results folder at: %s" % args.results)
  308. return 1
  309. if not os.path.exists(os.path.join(args.results, "newfailed")):
  310. print(
  311. "Could not find an failed tests in the results folder at: %s" % args.results
  312. )
  313. return 1
  314. print("Pre-flight checks passed")
  315. suites = []
  316. tmp = []
  317. print("Loading failed tests")
  318. try:
  319. with open(os.path.join(args.results, "newfailed"), "r") as ffile:
  320. tmp = json.loads(ffile.read())
  321. except json.decoder.JSONDecodeError:
  322. print("File containing the failed tests is not JSON")
  323. return 1
  324. for suite in tmp:
  325. if suite.startswith(("py2-", "py3-")):
  326. suites.append(suite[4:])
  327. return _run_test_suites(args, set(suites))
  328. def _get_pyvers(args):
  329. pyvers = [2, 3]
  330. if args.py2:
  331. pyvers = [2,]
  332. elif args.py3:
  333. pyvers = [3,]
  334. un_versioned = False
  335. try:
  336. subprocess.check_call(["which", RUNNER_PY])
  337. un_versioned = True
  338. except subprocess.CalledProcessError:
  339. print("No %s found no unversioned runner" % RUNNER_PY)
  340. if 2 in pyvers:
  341. nopy2 = False
  342. try:
  343. subprocess.check_call(["which", RUNNER_PY2])
  344. except subprocess.CalledProcessError:
  345. print("No %s found, removing python 2" % RUNNER_PY2)
  346. del pyvers[pyvers.index(2)]
  347. if 3 in pyvers:
  348. nopy3 = False
  349. try:
  350. subprocess.check_call(["which", RUNNER_PY3])
  351. except subprocess.CalledProcessError:
  352. print("No %s found, removing python 3" % RUNNER_PY3)
  353. del pyvers[pyvers.index(3)]
  354. if not pyvers and un_versioned:
  355. pyvers = [""]
  356. return pyvers
  357. def _run_test_suites(args, suites):
  358. print("Using %d processes" % NUMPROCS)
  359. print("Start timing")
  360. start = time.time()
  361. global PRINTLOCK
  362. PRINTLOCK = threading.RLock()
  363. global NUMREMAINING
  364. NUMREMAINING = 0
  365. sem = threading.BoundedSemaphore(NUMPROCS)
  366. # Create a worker per test
  367. workers = {}
  368. pyvers = _get_pyvers(args)
  369. if not pyvers:
  370. return 1
  371. if len(pyvers) == 1:
  372. if pyvers[0] == 2:
  373. subprocess.check_call([
  374. "sed", "-i", "-e", "s|python|python2|",
  375. "pagure/hooks/files/hookrunner"
  376. ])
  377. subprocess.check_call([
  378. "sed", "-i", "-e", "s|['alembic',|['alembic-2',|",
  379. "tests/test_alembic.py"
  380. ])
  381. elif pyvers[0] == 3:
  382. subprocess.check_call([
  383. "sed", "-i", "-e", "s|python|python3|",
  384. "pagure/hooks/files/hookrunner"
  385. ], cwd=HERE)
  386. subprocess.check_call([
  387. "sed", "-i", "-e", "s|\['alembic',|\['alembic-3',|",
  388. "tests/test_alembic.py"
  389. ], cwd=HERE)
  390. for suite in suites:
  391. for pyver in pyvers:
  392. NUMREMAINING += 1
  393. workers["py%s-%s" % (pyver, suite)] = WorkerThread(
  394. sem, pyver, suite, args.results, args.with_coverage
  395. )
  396. # Start the workers
  397. print("Starting the workers")
  398. print()
  399. print()
  400. for worker in workers.values():
  401. worker.start()
  402. # Wait for them to terminate
  403. for worker in workers:
  404. workers[worker].join()
  405. print_running()
  406. print()
  407. print("All work done")
  408. subprocess.check_call([
  409. "git",
  410. "checkout",
  411. "pagure/hooks/files/hookrunner",
  412. "tests/test_alembic.py"
  413. ])
  414. # Gather results
  415. print()
  416. print()
  417. if FAILED:
  418. print("Failed tests:")
  419. for worker in workers:
  420. if not workers[worker].failed:
  421. continue
  422. print("FAILED test: %s" % (worker))
  423. # Write failed
  424. if FAILED:
  425. with open(os.path.join(args.results, "newfailed"), "w") as ffile:
  426. ffile.write(json.dumps(FAILED))
  427. # Exit
  428. outcode = 0
  429. if len(FAILED) == 0:
  430. print("ALL PASSED! CONGRATULATIONS!")
  431. else:
  432. outcode = 1
  433. # Stats
  434. end = time.time()
  435. print()
  436. print()
  437. print(
  438. "Ran %d tests in %f seconds, of which %d failed"
  439. % (len(workers), (end - start), len(FAILED))
  440. )
  441. if outcode == 0 and args.with_coverage:
  442. do_show_coverage(args)
  443. return outcode
  444. def do_list(args):
  445. """ List tests that failed the last/specified run.
  446. """
  447. # Some pre-flight checks
  448. if not os.path.exists("./.git") or not os.path.exists("./pagure"):
  449. print("Please run from a single level into the Pagure codebase")
  450. return 1
  451. if not os.path.exists(args.results):
  452. print("Could not find an existing results folder at: %s" % args.results)
  453. return 1
  454. if not os.path.exists(os.path.join(args.results, "newfailed")):
  455. print(
  456. "Could not find an failed tests in the results folder at: %s" % args.results
  457. )
  458. return 1
  459. print("Pre-flight checks passed")
  460. suites = []
  461. tmp = []
  462. print("Loading failed tests")
  463. try:
  464. with open(os.path.join(args.results, "newfailed"), "r") as ffile:
  465. suites = json.loads(ffile.read())
  466. except json.decoder.JSONDecodeError:
  467. print("File containing the failed tests is not JSON")
  468. return 1
  469. print("Failed tests")
  470. failed_tests = len(suites)
  471. if args.n:
  472. suites = suites[:args.n]
  473. print("- " + "\n- ".join(suites))
  474. print("Total: %s test failed" % failed_tests)
  475. if args.show:
  476. for suite in suites:
  477. cmd = ["less", os.path.join(args.results, suite)]
  478. subprocess.check_call(cmd)
  479. def do_show_coverage(args):
  480. print()
  481. print("Combining coverage results...")
  482. pyvers = _get_pyvers(args)
  483. for pyver in pyvers:
  484. coverfiles = []
  485. for fname in os.listdir(args.results):
  486. if fname.endswith(".coverage") and fname.startswith("py%s-" % pyver):
  487. coverfiles.append(os.path.join(args.results, fname))
  488. cover = None
  489. if pyver == 2:
  490. cover = COVER_PY2
  491. elif pyver == 3:
  492. cover = COVER_PY3
  493. else:
  494. cover = COVER_PY
  495. env = {"COVERAGE_FILE": os.path.join(args.results, "combined.coverage")}
  496. cmd = [cover, "combine"] + coverfiles
  497. subprocess.check_call(cmd, env=env)
  498. print()
  499. print("Python %s coverage: " % pyver)
  500. cmd = [cover, "report", "--include=./pagure/*", "-m"]
  501. subprocess.check_call(cmd, env=env)
  502. def main():
  503. """ Main function """
  504. # Set up parser for global args
  505. parser = setup_parser()
  506. # Parse the commandline
  507. try:
  508. arg = parser.parse_args()
  509. except argparse.ArgumentTypeError as err:
  510. print("\nError: {0}".format(err))
  511. return 2
  512. logging.basicConfig()
  513. if arg.debug:
  514. LOG.setLevel(logging.DEBUG)
  515. if "func" not in arg:
  516. parser.print_help()
  517. return 1
  518. arg.results = os.path.abspath(arg.results)
  519. return_code = 0
  520. try:
  521. return_code = arg.func(arg)
  522. except KeyboardInterrupt:
  523. print("\nInterrupted by user.")
  524. return_code = 1
  525. except Exception as err:
  526. print("Error: {0}".format(err))
  527. logging.exception("Generic error caught:")
  528. return_code = 5
  529. return return_code
  530. if __name__ == "__main__":
  531. sys.exit(main())