1
0

dev-data.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. """ Populate the pagure db with some dev data. """
  4. from __future__ import print_function, unicode_literals, absolute_import
  5. import argparse
  6. import os
  7. import tempfile
  8. import pygit2
  9. import shutil
  10. import six
  11. from sqlalchemy import create_engine, MetaData
  12. import pagure
  13. import tests
  14. import pagure.lib.model
  15. import pagure.lib.query
  16. from pagure.lib.login import generate_hashed_value
  17. from pagure.lib.model import create_default_status
  18. from pagure.lib.repo import PagureRepo
  19. """
  20. Usage:
  21. python dev-data.py --init
  22. python dev-data.py --clean
  23. python dev-data.py --populate
  24. python dev-data.py --all
  25. """
  26. _config = pagure.config.reload_config()
  27. def empty_dev_db(session):
  28. print("")
  29. print("WARNING: Deleting all data from", _config["DB_URL"])
  30. response = os.environ.get("FORCE_DELETE")
  31. if not response:
  32. response = six.moves.input("Do you want to continue? (yes/no) ")
  33. if response.lower().startswith("y"):
  34. tables = reversed(pagure.lib.model_base.BASE.metadata.sorted_tables)
  35. for tbl in tables:
  36. session.execute(tbl.delete())
  37. else:
  38. exit("Aborting.")
  39. def insert_data(session, username, user_email):
  40. _config["EMAIL_SEND"] = False
  41. _config["TESTING"] = True
  42. ######################################
  43. # tags
  44. item = pagure.lib.model.Tag(tag="tag1")
  45. session.add(item)
  46. session.commit()
  47. ######################################
  48. # Users
  49. # Create a couple of users
  50. pingou = item = pagure.lib.model.User(
  51. user="pingou",
  52. fullname="PY C",
  53. password=generate_hashed_value("testing123"),
  54. token=None,
  55. default_email="bar@pingou.com",
  56. )
  57. session.add(item)
  58. session.commit()
  59. print(
  60. "User created: {} <{}>, {}".format(
  61. item.user, item.default_email, "testing123"
  62. )
  63. )
  64. foo = item = pagure.lib.model.User(
  65. user="foo",
  66. fullname="foo bar",
  67. password=generate_hashed_value("testing123"),
  68. token=None,
  69. default_email="foo@bar.com",
  70. )
  71. session.add(item)
  72. session.commit()
  73. print(
  74. "User created: {} <{}>, {}".format(
  75. item.user, item.default_email, "testing123"
  76. )
  77. )
  78. you = item = pagure.lib.model.User(
  79. user=username,
  80. fullname=username,
  81. password=generate_hashed_value("testing123"),
  82. token=None,
  83. default_email=user_email,
  84. )
  85. session.add(item)
  86. session.commit()
  87. print(
  88. "User created: {} <{}>, {}".format(
  89. item.user, item.default_email, "testing123"
  90. )
  91. )
  92. ######################################
  93. # pagure_group
  94. item = pagure.lib.model.PagureGroup(
  95. group_name="admin",
  96. group_type="admin",
  97. user_id=pingou.id,
  98. display_name="admin",
  99. description="Admin Group",
  100. )
  101. session.add(item)
  102. session.commit()
  103. print('Created "admin" group. Pingou is a member.')
  104. # Add a couple of groups so that we can list them
  105. item = pagure.lib.model.PagureGroup(
  106. group_name="group",
  107. group_type="user",
  108. user_id=pingou.id,
  109. display_name="group group",
  110. description="this is a group group",
  111. )
  112. session.add(item)
  113. session.commit()
  114. print('Created "group" group. Pingou is a member.')
  115. item = pagure.lib.model.PagureGroup(
  116. group_name="rel-eng",
  117. group_type="user",
  118. user_id=pingou.id,
  119. display_name="Release Engineering",
  120. description="The group of release engineers",
  121. )
  122. session.add(item)
  123. session.commit()
  124. print('Created "rel-eng" group. Pingou is a member.')
  125. ######################################
  126. # projects
  127. import shutil
  128. # delete folder from local instance to start from a clean slate
  129. if os.path.exists(_config["GIT_FOLDER"]):
  130. shutil.rmtree(_config["GIT_FOLDER"])
  131. # Create projects
  132. item = project1 = pagure.lib.model.Project(
  133. user_id=pingou.id,
  134. name="test",
  135. is_fork=False,
  136. parent_id=None,
  137. description="test project #1",
  138. hook_token="aaabbbccc",
  139. )
  140. item.close_status = ["Invalid", "Insufficient data", "Fixed", "Duplicate"]
  141. session.add(item)
  142. session.flush()
  143. tests.create_locks(session, item)
  144. item = project2 = pagure.lib.model.Project(
  145. user_id=pingou.id,
  146. name="test2",
  147. is_fork=False,
  148. parent_id=None,
  149. description="test project #2",
  150. hook_token="aaabbbddd",
  151. )
  152. item.close_status = ["Invalid", "Insufficient data", "Fixed", "Duplicate"]
  153. session.add(item)
  154. item = project3 = pagure.lib.model.Project(
  155. user_id=pingou.id,
  156. name="test3",
  157. is_fork=False,
  158. parent_id=None,
  159. description="namespaced test project",
  160. hook_token="aaabbbeee",
  161. namespace="somenamespace",
  162. )
  163. item.close_status = ["Invalid", "Insufficient data", "Fixed", "Duplicate"]
  164. session.add(item)
  165. session.commit()
  166. tests.create_projects_git(_config["GIT_FOLDER"], bare=True)
  167. add_content_git_repo(os.path.join(_config["GIT_FOLDER"], "test.git"))
  168. tests.add_readme_git_repo(os.path.join(_config["GIT_FOLDER"], "test.git"))
  169. # Add some content to the git repo
  170. add_content_git_repo(
  171. os.path.join(_config["GIT_FOLDER"], "forks", "pingou", "test.git")
  172. )
  173. tests.add_readme_git_repo(
  174. os.path.join(_config["GIT_FOLDER"], "forks", "pingou", "test.git")
  175. )
  176. tests.add_commit_git_repo(
  177. os.path.join(_config["GIT_FOLDER"], "forks", "pingou", "test.git"),
  178. ncommits=10,
  179. )
  180. ######################################
  181. # user_emails
  182. item = pagure.lib.model.UserEmail(
  183. user_id=pingou.id, email="bar@pingou.com"
  184. )
  185. session.add(item)
  186. item = pagure.lib.model.UserEmail(
  187. user_id=pingou.id, email="foo@pingou.com"
  188. )
  189. session.add(item)
  190. item = pagure.lib.model.UserEmail(user_id=foo.id, email="foo@bar.com")
  191. session.add(item)
  192. item = pagure.lib.model.UserEmail(user_id=you.id, email=user_email)
  193. session.add(item)
  194. session.commit()
  195. ######################################
  196. # user_emails_pending
  197. email_pend = pagure.lib.model.UserEmailPending(
  198. user_id=pingou.id, email="foo@fp.o", token="abcdef"
  199. )
  200. session.add(email_pend)
  201. session.commit()
  202. ######################################
  203. # issues
  204. # Add an issue and tag it so that we can list them
  205. item = pagure.lib.model.Issue(
  206. id=1001,
  207. uid="foobar",
  208. project_id=project1.id,
  209. title="Problem with jenkins build",
  210. content="For some reason the tests fail at line:24",
  211. user_id=pingou.id,
  212. )
  213. session.add(item)
  214. session.commit()
  215. item = pagure.lib.model.Issue(
  216. id=1002,
  217. uid="foobar2",
  218. project_id=project1.id,
  219. title="Unit tests failing",
  220. content="Need to fix code for the unit tests to "
  221. "pass so jenkins build can complete.",
  222. user_id=pingou.id,
  223. )
  224. session.add(item)
  225. session.commit()
  226. item = pagure.lib.model.Issue(
  227. id=1003,
  228. uid="foobar3",
  229. project_id=project1.id,
  230. title="Segfault during execution",
  231. content="Index out of bounds for variable i?",
  232. user_id=you.id,
  233. )
  234. session.add(item)
  235. session.commit()
  236. ######################################
  237. # pagure_user_group
  238. group = pagure.lib.query.search_groups(
  239. session, pattern=None, group_name="rel-eng", group_type=None
  240. )
  241. item = pagure.lib.model.PagureUserGroup(
  242. user_id=pingou.id, group_id=group.id
  243. )
  244. session.add(item)
  245. session.commit()
  246. group = pagure.lib.query.search_groups(
  247. session, pattern=None, group_name="admin", group_type=None
  248. )
  249. item = pagure.lib.model.PagureUserGroup(user_id=you.id, group_id=group.id)
  250. session.add(item)
  251. session.commit()
  252. group = pagure.lib.query.search_groups(
  253. session, pattern=None, group_name="group", group_type=None
  254. )
  255. item = pagure.lib.model.PagureUserGroup(user_id=foo.id, group_id=group.id)
  256. session.add(item)
  257. session.commit()
  258. ######################################
  259. # projects_groups
  260. group = pagure.lib.query.search_groups(
  261. session, pattern=None, group_name="rel-eng", group_type=None
  262. )
  263. repo = pagure.lib.query.get_authorized_project(session, "test")
  264. item = pagure.lib.model.ProjectGroup(
  265. project_id=repo.id, group_id=group.id, access="commit"
  266. )
  267. session.add(item)
  268. session.commit()
  269. group = pagure.lib.query.search_groups(
  270. session, pattern=None, group_name="admin", group_type=None
  271. )
  272. repo = pagure.lib.query.get_authorized_project(session, "test2")
  273. item = pagure.lib.model.ProjectGroup(
  274. project_id=repo.id, group_id=group.id, access="admin"
  275. )
  276. session.add(item)
  277. session.commit()
  278. ######################################
  279. # pull_requests
  280. repo = pagure.lib.query.get_authorized_project(session, "test")
  281. forked_repo = pagure.lib.query.get_authorized_project(session, "test")
  282. req = pagure.lib.query.new_pull_request(
  283. session=session,
  284. repo_from=forked_repo,
  285. branch_from="master",
  286. repo_to=repo,
  287. branch_to="master",
  288. title="Fixing code for unittest",
  289. user=username,
  290. status="Open",
  291. )
  292. session.commit()
  293. repo = pagure.lib.query.get_authorized_project(session, "test")
  294. forked_repo = pagure.lib.query.get_authorized_project(session, "test")
  295. req = pagure.lib.query.new_pull_request(
  296. session=session,
  297. repo_from=forked_repo,
  298. branch_from="master",
  299. repo_to=repo,
  300. branch_to="master",
  301. title="add very nice README",
  302. user=username,
  303. status="Open",
  304. )
  305. session.commit()
  306. repo = pagure.lib.query.get_authorized_project(session, "test")
  307. forked_repo = pagure.lib.query.get_authorized_project(session, "test")
  308. req = pagure.lib.query.new_pull_request(
  309. session=session,
  310. repo_from=forked_repo,
  311. branch_from="master",
  312. repo_to=repo,
  313. branch_to="master",
  314. title="Add README",
  315. user=username,
  316. status="Closed",
  317. )
  318. session.commit()
  319. repo = pagure.lib.query.get_authorized_project(session, "test")
  320. forked_repo = pagure.lib.query.get_authorized_project(session, "test")
  321. req = pagure.lib.query.new_pull_request(
  322. session=session,
  323. repo_from=forked_repo,
  324. branch_from="master",
  325. repo_to=repo,
  326. branch_to="master",
  327. title="Fix some containers",
  328. user=username,
  329. status="Merged",
  330. )
  331. session.commit()
  332. repo = pagure.lib.query.get_authorized_project(session, "test")
  333. forked_repo = pagure.lib.query.get_authorized_project(session, "test")
  334. req = pagure.lib.query.new_pull_request(
  335. session=session,
  336. repo_from=forked_repo,
  337. branch_from="master",
  338. repo_to=repo,
  339. branch_to="master",
  340. title="Fix pull request statuses",
  341. user=username,
  342. status="Closed",
  343. )
  344. session.commit()
  345. repo = pagure.lib.query.get_authorized_project(session, "test")
  346. forked_repo = pagure.lib.query.get_authorized_project(session, "test")
  347. req = pagure.lib.query.new_pull_request(
  348. session=session,
  349. repo_from=forked_repo,
  350. branch_from="master",
  351. repo_to=repo,
  352. branch_to="master",
  353. title="Fixing UI of issue",
  354. user=username,
  355. status="Merged",
  356. )
  357. session.commit()
  358. #####################################
  359. # tokens
  360. tests.create_tokens(session, user_id=pingou.id, project_id=project1.id)
  361. ######################################
  362. # user_projects
  363. repo = pagure.lib.query.get_authorized_project(session, "test")
  364. item = pagure.lib.model.ProjectUser(
  365. project_id=repo.id, user_id=foo.id, access="commit"
  366. )
  367. session.add(item)
  368. session.commit()
  369. repo = pagure.lib.query.get_authorized_project(session, "test2")
  370. item = pagure.lib.model.ProjectUser(
  371. project_id=repo.id, user_id=you.id, access="commit"
  372. )
  373. session.add(item)
  374. session.commit()
  375. ######################################
  376. # issue_comments
  377. item = pagure.lib.model.IssueComment(
  378. user_id=pingou.id,
  379. issue_uid="foobar",
  380. comment="We may need to adjust the unittests instead of the code.",
  381. )
  382. session.add(item)
  383. session.commit()
  384. ######################################
  385. # issue_to_issue
  386. repo = pagure.lib.query.get_authorized_project(session, "test")
  387. all_issues = pagure.lib.query.search_issues(session, repo)
  388. pagure.lib.query.add_issue_dependency(
  389. session, all_issues[0], all_issues[1], "pingou"
  390. )
  391. ######################################
  392. # pull_request_comments
  393. user = pagure.lib.query.search_user(session, username="pingou")
  394. # only 1 pull request available atm
  395. pr = pagure.lib.query.get_pull_request_of_user(session, "pingou")[0]
  396. item = pagure.lib.model.PullRequestComment(
  397. pull_request_uid=pr.uid,
  398. user_id=user.id,
  399. comment="+1 for me. Btw, could you rebase before you merge?",
  400. notification=0,
  401. )
  402. session.add(item)
  403. session.commit()
  404. ######################################
  405. # pull_request_flags
  406. # only 1 pull request available atm
  407. pr = pagure.lib.query.get_pull_request_of_user(session, "pingou")[0]
  408. item = pagure.lib.model.PullRequestFlag(
  409. uid="random_pr_flag_uid",
  410. pull_request_uid=pr.uid,
  411. user_id=pingou.id,
  412. username=pingou.user,
  413. percent=80,
  414. comment="Jenkins build passes",
  415. url=str(pr.id),
  416. status="success",
  417. )
  418. session.add(item)
  419. session.commit()
  420. pr = pagure.lib.query.get_pull_request_of_user(session, "foo")[1]
  421. item = pagure.lib.model.PullRequestFlag(
  422. uid="oink oink uid",
  423. pull_request_uid=pr.uid,
  424. user_id=pingou.id,
  425. username=pingou.user,
  426. percent=80,
  427. comment="Jenkins does not pass",
  428. url=str(pr.id),
  429. status="failure",
  430. )
  431. session.add(item)
  432. session.commit()
  433. ######################################
  434. # pull_request_assignee
  435. pr = pagure.lib.query.search_pull_requests(session, requestid="1006")
  436. pr.assignee_id = pingou.id
  437. session.commit()
  438. pr = pagure.lib.query.search_pull_requests(session, requestid="1007")
  439. pr.assignee_id = you.id
  440. session.commit()
  441. pr = pagure.lib.query.search_pull_requests(session, requestid="1004")
  442. pr.assignee_id = foo.id
  443. session.commit()
  444. ######################################
  445. # tags_issues
  446. repo = pagure.lib.query.get_authorized_project(session, "test")
  447. issues = pagure.lib.query.search_issues(session, repo)
  448. item = pagure.lib.model.TagIssue(issue_uid=issues[0].uid, tag="tag1")
  449. session.add(item)
  450. session.commit()
  451. ######################################
  452. # tokens_acls
  453. tests.create_tokens_acl(session)
  454. ######################################
  455. # Fork a project
  456. # delete fork data
  457. fork_proj_location = "forks/foo/test.git"
  458. try:
  459. shutil.rmtree(os.path.join(_config["GIT_FOLDER"], fork_proj_location))
  460. except:
  461. print("git folder already deleted")
  462. try:
  463. shutil.rmtree(os.path.join(_config["DOCS_FOLDER"], fork_proj_location))
  464. except:
  465. print("docs folder already deleted")
  466. try:
  467. shutil.rmtree(
  468. os.path.join(_config["TICKETS_FOLDER"], fork_proj_location)
  469. )
  470. except:
  471. print("tickets folder already deleted")
  472. try:
  473. shutil.rmtree(
  474. os.path.join(_config["REQUESTS_FOLDER"], fork_proj_location)
  475. )
  476. except:
  477. print("requests folder already deleted")
  478. repo = pagure.lib.query.get_authorized_project(session, "test")
  479. result = pagure.lib.query.fork_project(session, "foo", repo)
  480. if result == 'Repo "test" cloned to "foo/test"':
  481. session.commit()
  482. def add_content_git_repo(folder, branch="master"):
  483. """ Create some content for the specified git repo. """
  484. if not os.path.exists(folder):
  485. os.makedirs(folder)
  486. brepo = pygit2.init_repository(folder, bare=True)
  487. newfolder = tempfile.mkdtemp(prefix="pagure-tests")
  488. repo = pygit2.clone_repository(folder, newfolder)
  489. # Create a file in that git repo
  490. with open(os.path.join(newfolder, "sources"), "w") as stream:
  491. stream.write("foo\n bar")
  492. repo.index.add("sources")
  493. repo.index.write()
  494. parents = []
  495. commit = None
  496. try:
  497. commit = repo.revparse_single("HEAD" if branch == "master" else branch)
  498. except KeyError:
  499. pass
  500. if commit:
  501. parents = [commit.oid.hex]
  502. # Commits the files added
  503. tree = repo.index.write_tree()
  504. author = pygit2.Signature("Alice Author", "alice@authors.tld")
  505. committer = pygit2.Signature("Cecil Committer", "cecil@committers.tld")
  506. repo.create_commit(
  507. "refs/heads/%s" % branch, # the name of the reference to update
  508. author,
  509. committer,
  510. "Add sources file for testing",
  511. # binary string representing the tree object ID
  512. tree,
  513. # list of binary strings representing parents of the new commit
  514. parents,
  515. )
  516. parents = []
  517. commit = None
  518. try:
  519. commit = repo.revparse_single("HEAD" if branch == "master" else branch)
  520. except KeyError:
  521. pass
  522. if commit:
  523. parents = [commit.oid.hex]
  524. subfolder = os.path.join("folder1", "folder2")
  525. if not os.path.exists(os.path.join(newfolder, subfolder)):
  526. os.makedirs(os.path.join(newfolder, subfolder))
  527. # Create a file in that git repo
  528. with open(os.path.join(newfolder, subfolder, "file"), "w") as stream:
  529. stream.write("foo\n bar\nbaz")
  530. repo.index.add(os.path.join(subfolder, "file"))
  531. repo.index.write()
  532. # Commits the files added
  533. tree = repo.index.write_tree()
  534. author = pygit2.Signature("Alice Author", "alice@authors.tld")
  535. committer = pygit2.Signature("Cecil Committer", "cecil@committers.tld")
  536. repo.create_commit(
  537. "refs/heads/%s" % branch, # the name of the reference to update
  538. author,
  539. committer,
  540. "Add some directory and a file for more testing",
  541. # binary string representing the tree object ID
  542. tree,
  543. # list of binary strings representing parents of the new commit
  544. parents,
  545. )
  546. # Push to origin
  547. ori_remote = repo.remotes[0]
  548. master_ref = repo.lookup_reference(
  549. "HEAD" if branch == "master" else "refs/heads/%s" % branch
  550. ).resolve()
  551. refname = "%s:%s" % (master_ref.name, master_ref.name)
  552. PagureRepo.push(ori_remote, refname)
  553. shutil.rmtree(newfolder)
  554. def _get_username():
  555. invalid_option = ["pingou", "foo"]
  556. user_name = os.environ.get("USER_NAME")
  557. if not user_name:
  558. print("")
  559. user_name = six.moves.input(
  560. "Enter your username so we can add you into the test data: "
  561. )
  562. cnt = 0
  563. while not user_name.strip() or user_name in invalid_option:
  564. print("Reserved names: " + str(invalid_option))
  565. user_name = six.moves.input(
  566. "Enter your username so we can add you into the " "test data: "
  567. )
  568. cnt += 1
  569. if cnt == 4:
  570. print("We asked too many times, bailing")
  571. sys.exit(1)
  572. return user_name
  573. def _get_user_email():
  574. invalid_option = ["bar@pingou.com", "foo@bar.com"]
  575. user_email = os.environ.get("USER_EMAIL")
  576. if not user_email:
  577. print("")
  578. user_email = six.moves.input("Enter your user email: ")
  579. cnt = 0
  580. while not user_email.strip() or user_email in invalid_option:
  581. print("Reserved names: " + str(invalid_option))
  582. user_email = six.moves.input("Enter your user email: ")
  583. cnt += 1
  584. if cnt == 4:
  585. print("We asked too many times, bailing")
  586. sys.exit(1)
  587. return user_email
  588. if __name__ == "__main__":
  589. desc = (
  590. "Run the dev database initialization/insertion/deletion "
  591. "script for db located " + str(_config["DB_URL"])
  592. )
  593. parser = argparse.ArgumentParser(prog="dev-data", description=desc)
  594. parser.add_argument(
  595. "-i", "--init", action="store_true", help="Create the dev db"
  596. )
  597. parser.add_argument(
  598. "-p", "--populate", action="store_true", help="Add test data to the db"
  599. )
  600. parser.add_argument(
  601. "-d", "--delete", action="store_true", help="Wipe the dev db"
  602. )
  603. parser.add_argument(
  604. "-a",
  605. "--all",
  606. action="store_true",
  607. help="Create, Populate then Wipe the dev db",
  608. )
  609. args = parser.parse_args()
  610. # forcing the user to choose
  611. if not any(vars(args).values()):
  612. parser.error("No arguments provided.")
  613. session = None
  614. if args.init or args.all:
  615. session = pagure.lib.model.create_tables(
  616. db_url=_config["DB_URL"],
  617. alembic_ini=None,
  618. acls=_config["ACLS"],
  619. debug=False,
  620. )
  621. print("Database created")
  622. if args.populate or args.all:
  623. if not session:
  624. session = pagure.lib.query.create_session(_config["DB_URL"])
  625. user_name = _get_username()
  626. user_email = _get_user_email()
  627. insert_data(session, user_name, user_email)
  628. if args.delete or args.all:
  629. empty_dev_db(session)