1
0

test_pagure_repospanner.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675
  1. # -*- coding: utf-8 -*-
  2. """
  3. (c) 2015-2018 - Copyright Red Hat Inc
  4. Authors:
  5. Patrick Uiterwijk <puiterwijk@redhat.com>
  6. """
  7. from __future__ import unicode_literals, absolute_import
  8. import datetime
  9. import functools
  10. import munch
  11. import unittest
  12. import shutil
  13. import subprocess
  14. import sys
  15. import tempfile
  16. import time
  17. import os
  18. import six
  19. import json
  20. import pygit2
  21. import requests
  22. from requests.adapters import HTTPAdapter
  23. from requests.packages.urllib3.util.retry import Retry
  24. from mock import patch, MagicMock
  25. sys.path.insert(
  26. 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
  27. )
  28. import pagure.lib.query
  29. import pagure.cli.admin
  30. import tests
  31. REPOSPANNER_CONFIG_TEMPLATE = """
  32. ---
  33. ca:
  34. path: %(path)s/repospanner/pki
  35. admin:
  36. url: https://localhost.localdomain:%(gitport)s/
  37. ca: %(path)s/repospanner/pki/ca.crt
  38. cert: %(path)s/repospanner/pki/admin.crt
  39. key: %(path)s/repospanner/pki/admin.key
  40. storage:
  41. state: %(path)s/repospanner/state
  42. git:
  43. type: tree
  44. clustered: true
  45. directory: %(path)s/repospanner/git
  46. listen:
  47. rpc: 127.0.0.1:%(rpcport)s
  48. http: 127.0.0.1:%(gitport)s
  49. certificates:
  50. ca: %(path)s/repospanner/pki/ca.crt
  51. client:
  52. cert: %(path)s/repospanner/pki/repospanner.localhost.crt
  53. key: %(path)s/repospanner/pki/repospanner.localhost.key
  54. server:
  55. default:
  56. cert: %(path)s/repospanner/pki/repospanner.localhost.crt
  57. key: %(path)s/repospanner/pki/repospanner.localhost.key
  58. hooks:
  59. debug: true
  60. bubblewrap:
  61. enabled: false
  62. unshare:
  63. - net
  64. - ipc
  65. - pid
  66. - uts
  67. share_net: false
  68. mount_proc: true
  69. mount_dev: true
  70. uid:
  71. gid:
  72. hostname: myhostname
  73. bind:
  74. ro_bind:
  75. - - /usr
  76. - /usr
  77. - - %(codepath)s
  78. - %(codepath)s
  79. - - %(path)s
  80. - %(path)s
  81. - - %(crosspath)s
  82. - %(crosspath)s
  83. symlink:
  84. - - usr/lib64
  85. - /lib64
  86. - - usr/bin
  87. - /bin
  88. runner: %(hookrunner_bin)s
  89. user: 0
  90. """
  91. class PagureRepoSpannerTests(tests.Modeltests):
  92. """ Tests for repoSpanner integration of pagure """
  93. repospanner_binary = None
  94. repospanner_runlog = None
  95. repospanner_proc = None
  96. def run_cacmd(self, logfile, *args):
  97. """ Run a repoSpanner CA command. """
  98. subprocess.check_call(
  99. [
  100. self.repospanner_binary,
  101. "--config",
  102. os.path.join(self.path, "repospanner", "config.yml"),
  103. # NEVER use this in a production system! It makes repeatable keys
  104. "ca",
  105. ]
  106. + list(args)
  107. + ["--very-insecure-weak-keys"],
  108. stdout=logfile,
  109. stderr=subprocess.STDOUT,
  110. )
  111. def setUp(self):
  112. """ set up the environment. """
  113. possible_paths = ["./repospanner", "/usr/bin/repospanner"]
  114. for option in possible_paths:
  115. option = os.path.abspath(option)
  116. if os.path.exists(option):
  117. self.repospanner_binary = option
  118. break
  119. if not self.repospanner_binary:
  120. raise unittest.SkipTest("repoSpanner not found")
  121. hookrunbins = [
  122. os.path.join(
  123. os.path.dirname(self.repospanner_binary), "repohookrunner"
  124. ),
  125. os.path.join("/usr", "libexec", "repohookrunner"),
  126. ]
  127. found = False
  128. for hookrunbin in hookrunbins:
  129. if os.path.exists(hookrunbin):
  130. found = True
  131. break
  132. if not found:
  133. raise Exception("repoSpanner found, but repohookrunner not")
  134. repobridgebins = [
  135. os.path.join(
  136. os.path.dirname(self.repospanner_binary), "repobridge"
  137. ),
  138. os.path.join("/usr", "libexec", "repobridge"),
  139. ]
  140. found = False
  141. for repobridgebin in repobridgebins:
  142. if os.path.exists(repobridgebin):
  143. found = True
  144. break
  145. if not found:
  146. raise Exception("repoSpanner found, but repobridge not")
  147. self.config_values["repobridge_binary"] = repobridgebin
  148. codepath = os.path.normpath(
  149. os.path.join(os.path.dirname(os.path.abspath(__file__)), "../")
  150. )
  151. # Only run the setUp() function if we are actually going ahead and run
  152. # this test. The reason being that otherwise, setUp will set up a
  153. # database, but because we "error out" from setUp, the tearDown()
  154. # function never gets called, leaving it behind.
  155. super(PagureRepoSpannerTests, self).setUp()
  156. # TODO: Find free ports
  157. configvals = {
  158. "path": self.path,
  159. "crosspath": self.dbfolder,
  160. "gitport": 8443 + sys.version_info.major,
  161. "rpcport": 8445 + sys.version_info.major,
  162. "codepath": codepath,
  163. "hookrunner_bin": hookrunbin,
  164. }
  165. os.mkdir(os.path.join(self.path, "repospanner"))
  166. cfgpath = os.path.join(self.path, "repospanner", "config.yml")
  167. with open(cfgpath, "w") as cfg:
  168. cfg.write(REPOSPANNER_CONFIG_TEMPLATE % configvals)
  169. with open(
  170. os.path.join(self.path, "repospanner", "keylog"), "w"
  171. ) as keylog:
  172. # Create the CA
  173. self.run_cacmd(
  174. keylog,
  175. "init",
  176. "localdomain",
  177. "--no-name-constraint",
  178. "--random-cn",
  179. )
  180. # Create the node cert
  181. self.run_cacmd(keylog, "node", "localhost", "repospanner")
  182. # Create the admin cert
  183. self.run_cacmd(
  184. keylog,
  185. "leaf",
  186. "admin",
  187. "--admin",
  188. "--region",
  189. "*",
  190. "--repo",
  191. "*",
  192. )
  193. # Create the Pagure cert
  194. self.run_cacmd(
  195. keylog,
  196. "leaf",
  197. "pagure",
  198. "--read",
  199. "--write",
  200. "--region",
  201. "*",
  202. "--repo",
  203. "*",
  204. )
  205. with open(
  206. os.path.join(self.path, "repospanner", "spawnlog"), "w"
  207. ) as spawnlog:
  208. # Initialize state
  209. subprocess.check_call(
  210. [
  211. self.repospanner_binary,
  212. "--config",
  213. cfgpath,
  214. "serve",
  215. "--spawn",
  216. ],
  217. stdout=spawnlog,
  218. stderr=subprocess.STDOUT,
  219. )
  220. self.repospanner_runlog = open(
  221. os.path.join(self.path, "repospanner", "runlog"), "w+"
  222. )
  223. try:
  224. self.repospanner_proc = subprocess.Popen(
  225. [
  226. self.repospanner_binary,
  227. "--config",
  228. cfgpath,
  229. "serve",
  230. "--debug",
  231. ],
  232. stdout=self.repospanner_runlog,
  233. stderr=subprocess.STDOUT,
  234. )
  235. except:
  236. # Make sure to clean up repoSpanner, since we did start it
  237. self.tearDown()
  238. raise
  239. attempts = 0
  240. while True:
  241. try:
  242. # Wait for the instance to become available
  243. resp = requests.get(
  244. "https://repospanner.localhost.localdomain:%d/"
  245. % configvals["gitport"],
  246. verify=os.path.join(
  247. self.path, "repospanner", "pki", "ca.crt"
  248. ),
  249. cert=(
  250. os.path.join(
  251. self.path, "repospanner", "pki", "pagure.crt"
  252. ),
  253. os.path.join(
  254. self.path, "repospanner", "pki", "pagure.key"
  255. ),
  256. ),
  257. )
  258. resp.raise_for_status()
  259. print("repoSpanner identification: %s" % resp.text)
  260. break
  261. except:
  262. if attempts < 5:
  263. attempts += 1
  264. time.sleep(1)
  265. continue
  266. # Make sure to clean up repoSpanner, since we did start it
  267. self.tearDown()
  268. raise
  269. # Upload the hook script to repoSpanner
  270. args = munch.Munch({"region": "default"})
  271. hookid = pagure.cli.admin.do_upload_repospanner_hooks(args)
  272. pagure.config.config["REPOSPANNER_REGIONS"]["default"]["hook"] = hookid
  273. def tearDown(self):
  274. """ Tear down the repoSpanner instance. """
  275. if self.repospanner_proc:
  276. # Tear down
  277. self.repospanner_proc.terminate()
  278. exitcode = self.repospanner_proc.wait()
  279. if exitcode != 0:
  280. print("repoSpanner exit code: %d" % exitcode)
  281. if self.repospanner_runlog:
  282. self.repospanner_runlog.close()
  283. super(PagureRepoSpannerTests, self).tearDown()
  284. def print_repospanner_log(fn):
  285. @functools.wraps(fn)
  286. def wrapper(self, *args, **kwargs):
  287. try:
  288. return fn(self, *args, **kwargs)
  289. finally:
  290. if self.repospanner_runlog:
  291. self.repospanner_runlog.seek(0, 0)
  292. print("repoSpanner log follows:")
  293. print(self.repospanner_runlog.read())
  294. return wrapper
  295. class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests):
  296. config_values = {
  297. "repospanner_new_repo": "'default'",
  298. "authbackend": "test_auth",
  299. }
  300. @print_repospanner_log
  301. @patch("pagure.ui.app.admin_session_timedout")
  302. def test_new_project(self, ast):
  303. """ Test creating a new repo by default on repoSpanner works. """
  304. ast.return_value = False
  305. user = tests.FakeUser(username="foo")
  306. with tests.user_set(self.app.application, user):
  307. output = self.app.get("/new/")
  308. self.assertEqual(output.status_code, 200)
  309. output_text = output.get_data(as_text=True)
  310. self.assertIn("<strong>Create new Project</strong>", output_text)
  311. data = {
  312. "name": "project-1",
  313. "description": "Project #1",
  314. "create_readme": "y",
  315. "csrf_token": self.get_csrf(),
  316. }
  317. output = self.app.post("/new/", data=data, follow_redirects=True)
  318. self.assertEqual(output.status_code, 200)
  319. output_text = output.get_data(as_text=True)
  320. self.assertIn(
  321. '<div class="projectinfo my-3">\nProject #1', output_text
  322. )
  323. self.assertIn(
  324. "<title>Overview - project-1 - Pagure</title>", output_text
  325. )
  326. self.assertIn("Added the README", output_text)
  327. output = self.app.get("/project-1/settings")
  328. self.assertIn(
  329. "This repository is on repoSpanner region default",
  330. output.get_data(as_text=True),
  331. )
  332. with tests.user_set(
  333. self.app.application, tests.FakeUser(username="pingou")
  334. ):
  335. # Verify that for forking, Git auth status is ignored (hooks should not be run)
  336. self.set_auth_status(False)
  337. data = {"csrf_token": self.get_csrf()}
  338. output = self.app.post(
  339. "/do_fork/project-1", data=data, follow_redirects=True
  340. )
  341. self.assertEqual(output.status_code, 200)
  342. output_text = output.get_data(as_text=True)
  343. self.assertIn(
  344. '<div class="projectinfo my-3">\nProject #1', output_text
  345. )
  346. self.assertIn(
  347. "<title>Overview - project-1 - Pagure</title>", output_text
  348. )
  349. self.assertIn("Added the README", output_text)
  350. self.assertIn(
  351. "/?next=http://localhost/fork/pingou/project-1", output_text
  352. )
  353. output = self.app.get("/fork/pingou/project-1/settings")
  354. self.assertIn(
  355. "This repository is on repoSpanner region default",
  356. output.get_data(as_text=True),
  357. )
  358. # Verify that only pseudo repos exist, and no on-disk repos got created
  359. repodirlist = os.listdir(os.path.join(self.path, "repos"))
  360. self.assertEqual(repodirlist, ["pseudo"])
  361. @print_repospanner_log
  362. @patch.dict(
  363. "pagure.config.config",
  364. {
  365. "ALLOW_HTTP_PULL_PUSH": True,
  366. "ALLOW_HTTP_PUSH": True,
  367. "HTTP_REPO_ACCESS_GITOLITE": False,
  368. },
  369. )
  370. def test_http_pull(self):
  371. """ Test that the HTTP pull endpoint works for repoSpanner. """
  372. tests.create_projects(self.session)
  373. tests.create_tokens(self.session)
  374. tests.create_tokens_acl(self.session)
  375. self.create_project_full("clonetest", {"create_readme": "y"})
  376. # Verify the new project is indeed on repoSpanner
  377. project = pagure.lib.query._get_project(self.session, "clonetest")
  378. self.assertTrue(project.is_on_repospanner)
  379. # Unfortunately, actually testing a git clone would need the app to
  380. # run on a TCP port, which the test environment doesn't do.
  381. output = self.app.get(
  382. "/clonetest.git/info/refs?service=git-upload-pack"
  383. )
  384. self.assertEqual(output.status_code, 200)
  385. self.assertEqual(
  386. output.content_type, "application/x-git-upload-pack-advertisement"
  387. )
  388. output_text = output.get_data(as_text=True)
  389. self.assertIn("# service=git-upload-pack", output_text)
  390. self.assertIn("agent=repoSpanner", output_text)
  391. self.assertIn("symref=HEAD:refs/heads/master", output_text)
  392. self.assertIn(" refs/heads/master\x00", output_text)
  393. output = self.app.post(
  394. "/clonetest.git/git-upload-pack",
  395. headers={"Content-Type": "application/x-git-upload-pack-request"},
  396. )
  397. self.assertEqual(output.status_code, 400)
  398. output_text = output.get_data(as_text=True)
  399. self.assertIn("Error processing your request", output_text)
  400. @print_repospanner_log
  401. @patch.dict(
  402. "pagure.config.config",
  403. {
  404. "ALLOW_HTTP_PULL_PUSH": True,
  405. "ALLOW_HTTP_PUSH": True,
  406. "HTTP_REPO_ACCESS_GITOLITE": False,
  407. },
  408. )
  409. def test_http_push(self):
  410. """ Test that the HTTP push endpoint works for repoSpanner. """
  411. tests.create_projects(self.session)
  412. tests.create_tokens(self.session)
  413. tests.create_tokens_acl(self.session)
  414. self.create_project_full("clonetest", {"create_readme": "y"})
  415. # Verify the new project is indeed on repoSpanner
  416. project = pagure.lib.query._get_project(self.session, "clonetest")
  417. self.assertTrue(project.is_on_repospanner)
  418. # Unfortunately, actually testing a git clone would need the app to
  419. # run on a TCP port, which the test environment doesn't do.
  420. output = self.app.get(
  421. "/clonetest.git/info/refs?service=git-receive-pack",
  422. environ_overrides={"REMOTE_USER": "pingou"},
  423. )
  424. self.assertEqual(output.status_code, 200)
  425. self.assertEqual(
  426. output.content_type, "application/x-git-receive-pack-advertisement"
  427. )
  428. output_text = output.get_data(as_text=True)
  429. self.assertIn("# service=git-receive-pack", output_text)
  430. self.assertIn("agent=repoSpanner", output_text)
  431. self.assertIn("symref=HEAD:refs/heads/master", output_text)
  432. self.assertIn(" refs/heads/master\x00", output_text)
  433. @print_repospanner_log
  434. @patch("pagure.ui.app.admin_session_timedout")
  435. def test_hooks(self, ast):
  436. """ Test hook setting and running works. """
  437. ast.return_value = False
  438. pagure.cli.admin.session = self.session
  439. user = tests.FakeUser(username="foo")
  440. with tests.user_set(self.app.application, user):
  441. data = {
  442. "name": "project-1",
  443. "description": "Project #1",
  444. "create_readme": "y",
  445. "csrf_token": self.get_csrf(),
  446. }
  447. output = self.app.post("/new/", data=data, follow_redirects=True)
  448. self.assertEqual(output.status_code, 200)
  449. output_text = output.get_data(as_text=True)
  450. self.assertIn(
  451. '<div class="projectinfo my-3">\nProject #1', output_text
  452. )
  453. self.assertIn(
  454. "<title>Overview - project-1 - Pagure</title>", output_text
  455. )
  456. self.assertIn("Added the README", output_text)
  457. output = self.app.get("/project-1/settings")
  458. self.assertIn(
  459. "This repository is on repoSpanner region default",
  460. output.get_data(as_text=True),
  461. )
  462. # Check file before the commit:
  463. output = self.app.get("/project-1/raw/master/f/README.md")
  464. self.assertEqual(output.status_code, 200)
  465. output_text = output.get_data(as_text=True)
  466. self.assertEqual(output_text, "# project-1\n\nProject #1")
  467. with tests.user_set(self.app.application, user):
  468. # Set editing Denied
  469. self.set_auth_status(False)
  470. # Try to make an edit in the repo
  471. data = {
  472. "content": "foo\n bar\n baz",
  473. "commit_title": "test commit",
  474. "commit_message": "Online commit",
  475. "email": "foo@bar.com",
  476. "branch": "master",
  477. "csrf_token": self.get_csrf(),
  478. }
  479. output = self.app.post(
  480. "/project-1/edit/master/f/README.md",
  481. data=data,
  482. follow_redirects=True,
  483. )
  484. self.assertEqual(output.status_code, 200)
  485. output_text = output.get_data(as_text=True)
  486. self.assertIn("Remote hook declined the push: ", output_text)
  487. self.assertIn(
  488. "Denied push for ref &#39;refs/heads/master&#39; for user &#39;foo&#39;",
  489. output_text,
  490. )
  491. # Check file after the commit:
  492. output = self.app.get("/project-1/raw/master/f/README.md")
  493. self.assertEqual(output.status_code, 200)
  494. output_text = output.get_data(as_text=True)
  495. self.assertEqual(output_text, "# project-1\n\nProject #1")
  496. # Set editing Allowed
  497. self.set_auth_status(True)
  498. # Try to make an edit in the repo
  499. data = {
  500. "content": "foo\n bar\n baz",
  501. "commit_title": "test commit",
  502. "commit_message": "Online commit",
  503. "email": "foo@bar.com",
  504. "branch": "master",
  505. "csrf_token": self.get_csrf(),
  506. }
  507. output = self.app.post(
  508. "/project-1/edit/master/f/README.md",
  509. data=data,
  510. follow_redirects=True,
  511. )
  512. self.assertEqual(output.status_code, 200)
  513. output_text = output.get_data(as_text=True)
  514. self.assertIn(
  515. "<title>Commits - project-1 - Pagure</title>", output_text
  516. )
  517. # Check file after the commit:
  518. output = self.app.get("/project-1/raw/master/f/README.md")
  519. self.assertEqual(output.status_code, 200)
  520. output_text = output.get_data(as_text=True)
  521. self.assertEqual(output_text, "foo\n bar\n baz")
  522. @print_repospanner_log
  523. @patch.dict(
  524. "pagure.config.config",
  525. {
  526. "PAGURE_ADMIN_USERS": ["pingou"],
  527. "ALLOW_ADMIN_IGNORE_EXISTING_REPOS": True,
  528. },
  529. )
  530. @patch("pagure.ui.app.admin_session_timedout")
  531. def test_adopt_project(self, ast):
  532. """ Test adopting a project in repoSpanner works. """
  533. ast.return_value = False
  534. user = tests.FakeUser(username="foo")
  535. with tests.user_set(self.app.application, user):
  536. output = self.app.get("/new/")
  537. self.assertEqual(output.status_code, 200)
  538. output_text = output.get_data(as_text=True)
  539. self.assertIn("<strong>Create new Project</strong>", output_text)
  540. data = {
  541. "name": "project-1",
  542. "description": "Project #1",
  543. "create_readme": "y",
  544. "csrf_token": self.get_csrf(),
  545. }
  546. output = self.app.post("/new/", data=data, follow_redirects=True)
  547. self.assertEqual(output.status_code, 200)
  548. output_text = output.get_data(as_text=True)
  549. self.assertIn(
  550. '<div class="projectinfo my-3">\nProject #1', output_text
  551. )
  552. self.assertIn(
  553. "<title>Overview - project-1 - Pagure</title>", output_text
  554. )
  555. self.assertIn("Added the README", output_text)
  556. output = self.app.get("/project-1/settings")
  557. self.assertIn(
  558. "This repository is on repoSpanner region default",
  559. output.get_data(as_text=True),
  560. )
  561. # Delete the project instance so that the actual repo remains
  562. project = pagure.lib.query._get_project(self.session, "project-1")
  563. self.session.delete(project)
  564. self.session.commit()
  565. shutil.rmtree(os.path.join(self.path, "repos", "pseudo"))
  566. user = tests.FakeUser(username="pingou")
  567. with tests.user_set(self.app.application, user):
  568. output = self.app.get("/project-1/")
  569. self.assertEqual(output.status_code, 404)
  570. data = {
  571. "name": "project-1",
  572. "description": "Recreated project #1",
  573. "create_readme": "false",
  574. "csrf_token": self.get_csrf(),
  575. }
  576. output = self.app.post("/new/", data=data, follow_redirects=True)
  577. self.assertEqual(output.status_code, 200)
  578. output_text = output.get_data(as_text=True)
  579. self.assertIn(
  580. "Repo pagure/main/project-1 already exists", output_text
  581. )
  582. data["ignore_existing_repos"] = "y"
  583. output = self.app.post("/new/", data=data, follow_redirects=True)
  584. self.assertEqual(output.status_code, 200)
  585. output_text = output.get_data(as_text=True)
  586. self.assertIn(
  587. '<div class="projectinfo my-3">\nRecreated project #1',
  588. output_text,
  589. )
  590. self.assertIn(
  591. "<title>Overview - project-1 - Pagure</title>", output_text
  592. )
  593. self.assertIn("Added the README", output_text)
  594. if __name__ == "__main__":
  595. unittest.main(verbosity=2)