#!/usr/bin/env python # -*- coding: utf-8 -*- """ Populate the pagure db with some dev data. """ from __future__ import print_function, unicode_literals, absolute_import import argparse import os import tempfile import pygit2 import shutil import six from sqlalchemy import create_engine, MetaData import pagure import tests import pagure.lib.model import pagure.lib.query from pagure.lib.login import generate_hashed_value from pagure.lib.model import create_default_status from pagure.lib.repo import PagureRepo """ Usage: python dev-data.py --init python dev-data.py --clean python dev-data.py --populate python dev-data.py --all """ _config = pagure.config.reload_config() def empty_dev_db(session): print("") print("WARNING: Deleting all data from", _config["DB_URL"]) response = os.environ.get("FORCE_DELETE") if not response: response = six.moves.input("Do you want to continue? (yes/no) ") if response.lower().startswith("y"): tables = reversed(pagure.lib.model_base.BASE.metadata.sorted_tables) for tbl in tables: session.execute(tbl.delete()) else: exit("Aborting.") def insert_data(session, username, user_email): _config["EMAIL_SEND"] = False _config["TESTING"] = True ###################################### # tags item = pagure.lib.model.Tag(tag="tag1") session.add(item) session.commit() ###################################### # Users # Create a couple of users pingou = item = pagure.lib.model.User( user="pingou", fullname="PY C", password=generate_hashed_value("testing123"), token=None, default_email="bar@pingou.com", ) session.add(item) session.commit() print( "User created: {} <{}>, {}".format( item.user, item.default_email, "testing123" ) ) foo = item = pagure.lib.model.User( user="foo", fullname="foo bar", password=generate_hashed_value("testing123"), token=None, default_email="foo@bar.com", ) session.add(item) session.commit() print( "User created: {} <{}>, {}".format( item.user, item.default_email, "testing123" ) ) you = item = pagure.lib.model.User( user=username, fullname=username, password=generate_hashed_value("testing123"), token=None, default_email=user_email, ) session.add(item) session.commit() print( "User created: {} <{}>, {}".format( item.user, item.default_email, "testing123" ) ) ###################################### # pagure_group item = pagure.lib.model.PagureGroup( group_name="admin", group_type="admin", user_id=pingou.id, display_name="admin", description="Admin Group", ) session.add(item) session.commit() print('Created "admin" group. Pingou is a member.') # Add a couple of groups so that we can list them item = pagure.lib.model.PagureGroup( group_name="group", group_type="user", user_id=pingou.id, display_name="group group", description="this is a group group", ) session.add(item) session.commit() print('Created "group" group. Pingou is a member.') item = pagure.lib.model.PagureGroup( group_name="rel-eng", group_type="user", user_id=pingou.id, display_name="Release Engineering", description="The group of release engineers", ) session.add(item) session.commit() print('Created "rel-eng" group. Pingou is a member.') ###################################### # projects import shutil # delete folder from local instance to start from a clean slate if os.path.exists(_config["GIT_FOLDER"]): shutil.rmtree(_config["GIT_FOLDER"]) # Create projects item = project1 = pagure.lib.model.Project( user_id=pingou.id, name="test", is_fork=False, parent_id=None, description="test project #1", hook_token="aaabbbccc", ) item.close_status = ["Invalid", "Insufficient data", "Fixed", "Duplicate"] session.add(item) session.flush() tests.create_locks(session, item) item = project2 = pagure.lib.model.Project( user_id=pingou.id, name="test2", is_fork=False, parent_id=None, description="test project #2", hook_token="aaabbbddd", ) item.close_status = ["Invalid", "Insufficient data", "Fixed", "Duplicate"] session.add(item) item = project3 = pagure.lib.model.Project( user_id=pingou.id, name="test3", is_fork=False, parent_id=None, description="namespaced test project", hook_token="aaabbbeee", namespace="somenamespace", ) item.close_status = ["Invalid", "Insufficient data", "Fixed", "Duplicate"] session.add(item) session.commit() tests.create_projects_git(_config["GIT_FOLDER"], bare=True) add_content_git_repo(os.path.join(_config["GIT_FOLDER"], "test.git")) tests.add_readme_git_repo(os.path.join(_config["GIT_FOLDER"], "test.git")) # Add some content to the git repo add_content_git_repo( os.path.join(_config["GIT_FOLDER"], "forks", "pingou", "test.git") ) tests.add_readme_git_repo( os.path.join(_config["GIT_FOLDER"], "forks", "pingou", "test.git") ) tests.add_commit_git_repo( os.path.join(_config["GIT_FOLDER"], "forks", "pingou", "test.git"), ncommits=10, ) ###################################### # user_emails item = pagure.lib.model.UserEmail( user_id=pingou.id, email="bar@pingou.com" ) session.add(item) item = pagure.lib.model.UserEmail( user_id=pingou.id, email="foo@pingou.com" ) session.add(item) item = pagure.lib.model.UserEmail(user_id=foo.id, email="foo@bar.com") session.add(item) item = pagure.lib.model.UserEmail(user_id=you.id, email=user_email) session.add(item) session.commit() ###################################### # user_emails_pending email_pend = pagure.lib.model.UserEmailPending( user_id=pingou.id, email="foo@fp.o", token="abcdef" ) session.add(email_pend) session.commit() ###################################### # issues # Add an issue and tag it so that we can list them item = pagure.lib.model.Issue( id=1001, uid="foobar", project_id=project1.id, title="Problem with jenkins build", content="For some reason the tests fail at line:24", user_id=pingou.id, ) session.add(item) session.commit() item = pagure.lib.model.Issue( id=1002, uid="foobar2", project_id=project1.id, title="Unit tests failing", content="Need to fix code for the unit tests to " "pass so jenkins build can complete.", user_id=pingou.id, ) session.add(item) session.commit() item = pagure.lib.model.Issue( id=1003, uid="foobar3", project_id=project1.id, title="Segfault during execution", content="Index out of bounds for variable i?", user_id=you.id, ) session.add(item) session.commit() ###################################### # pagure_user_group group = pagure.lib.query.search_groups( session, pattern=None, group_name="rel-eng", group_type=None ) item = pagure.lib.model.PagureUserGroup( user_id=pingou.id, group_id=group.id ) session.add(item) session.commit() group = pagure.lib.query.search_groups( session, pattern=None, group_name="admin", group_type=None ) item = pagure.lib.model.PagureUserGroup(user_id=you.id, group_id=group.id) session.add(item) session.commit() group = pagure.lib.query.search_groups( session, pattern=None, group_name="group", group_type=None ) item = pagure.lib.model.PagureUserGroup(user_id=foo.id, group_id=group.id) session.add(item) session.commit() ###################################### # projects_groups group = pagure.lib.query.search_groups( session, pattern=None, group_name="rel-eng", group_type=None ) repo = pagure.lib.query.get_authorized_project(session, "test") item = pagure.lib.model.ProjectGroup( project_id=repo.id, group_id=group.id, access="commit" ) session.add(item) session.commit() group = pagure.lib.query.search_groups( session, pattern=None, group_name="admin", group_type=None ) repo = pagure.lib.query.get_authorized_project(session, "test2") item = pagure.lib.model.ProjectGroup( project_id=repo.id, group_id=group.id, access="admin" ) session.add(item) session.commit() ###################################### # pull_requests repo = pagure.lib.query.get_authorized_project(session, "test") forked_repo = pagure.lib.query.get_authorized_project(session, "test") req = pagure.lib.query.new_pull_request( session=session, repo_from=forked_repo, branch_from="master", repo_to=repo, branch_to="master", title="Fixing code for unittest", user=username, status="Open", ) session.commit() repo = pagure.lib.query.get_authorized_project(session, "test") forked_repo = pagure.lib.query.get_authorized_project(session, "test") req = pagure.lib.query.new_pull_request( session=session, repo_from=forked_repo, branch_from="master", repo_to=repo, branch_to="master", title="add very nice README", user=username, status="Open", ) session.commit() repo = pagure.lib.query.get_authorized_project(session, "test") forked_repo = pagure.lib.query.get_authorized_project(session, "test") req = pagure.lib.query.new_pull_request( session=session, repo_from=forked_repo, branch_from="master", repo_to=repo, branch_to="master", title="Add README", user=username, status="Closed", ) session.commit() repo = pagure.lib.query.get_authorized_project(session, "test") forked_repo = pagure.lib.query.get_authorized_project(session, "test") req = pagure.lib.query.new_pull_request( session=session, repo_from=forked_repo, branch_from="master", repo_to=repo, branch_to="master", title="Fix some containers", user=username, status="Merged", ) session.commit() repo = pagure.lib.query.get_authorized_project(session, "test") forked_repo = pagure.lib.query.get_authorized_project(session, "test") req = pagure.lib.query.new_pull_request( session=session, repo_from=forked_repo, branch_from="master", repo_to=repo, branch_to="master", title="Fix pull request statuses", user=username, status="Closed", ) session.commit() repo = pagure.lib.query.get_authorized_project(session, "test") forked_repo = pagure.lib.query.get_authorized_project(session, "test") req = pagure.lib.query.new_pull_request( session=session, repo_from=forked_repo, branch_from="master", repo_to=repo, branch_to="master", title="Fixing UI of issue", user=username, status="Merged", ) session.commit() ##################################### # tokens tests.create_tokens(session, user_id=pingou.id, project_id=project1.id) ###################################### # user_projects repo = pagure.lib.query.get_authorized_project(session, "test") item = pagure.lib.model.ProjectUser( project_id=repo.id, user_id=foo.id, access="commit" ) session.add(item) session.commit() repo = pagure.lib.query.get_authorized_project(session, "test2") item = pagure.lib.model.ProjectUser( project_id=repo.id, user_id=you.id, access="commit" ) session.add(item) session.commit() ###################################### # issue_comments item = pagure.lib.model.IssueComment( user_id=pingou.id, issue_uid="foobar", comment="We may need to adjust the unittests instead of the code.", ) session.add(item) session.commit() ###################################### # issue_to_issue repo = pagure.lib.query.get_authorized_project(session, "test") all_issues = pagure.lib.query.search_issues(session, repo) pagure.lib.query.add_issue_dependency( session, all_issues[0], all_issues[1], "pingou" ) ###################################### # pull_request_comments user = pagure.lib.query.search_user(session, username="pingou") # only 1 pull request available atm pr = pagure.lib.query.get_pull_request_of_user(session, "pingou")[0] item = pagure.lib.model.PullRequestComment( pull_request_uid=pr.uid, user_id=user.id, comment="+1 for me. Btw, could you rebase before you merge?", notification=0, ) session.add(item) session.commit() ###################################### # pull_request_flags # only 1 pull request available atm pr = pagure.lib.query.get_pull_request_of_user(session, "pingou")[0] item = pagure.lib.model.PullRequestFlag( uid="random_pr_flag_uid", pull_request_uid=pr.uid, user_id=pingou.id, username=pingou.user, percent=80, comment="Jenkins build passes", url=str(pr.id), status="success", ) session.add(item) session.commit() pr = pagure.lib.query.get_pull_request_of_user(session, "foo")[1] item = pagure.lib.model.PullRequestFlag( uid="oink oink uid", pull_request_uid=pr.uid, user_id=pingou.id, username=pingou.user, percent=80, comment="Jenkins does not pass", url=str(pr.id), status="failure", ) session.add(item) session.commit() ###################################### # pull_request_assignee pr = pagure.lib.query.search_pull_requests(session, requestid="1006") pr.assignee_id = pingou.id session.commit() pr = pagure.lib.query.search_pull_requests(session, requestid="1007") pr.assignee_id = you.id session.commit() pr = pagure.lib.query.search_pull_requests(session, requestid="1004") pr.assignee_id = foo.id session.commit() ###################################### # tags_issues repo = pagure.lib.query.get_authorized_project(session, "test") issues = pagure.lib.query.search_issues(session, repo) item = pagure.lib.model.TagIssue(issue_uid=issues[0].uid, tag="tag1") session.add(item) session.commit() ###################################### # tokens_acls tests.create_tokens_acl(session) ###################################### # Fork a project # delete fork data fork_proj_location = "forks/foo/test.git" try: shutil.rmtree(os.path.join(_config["GIT_FOLDER"], fork_proj_location)) except: print("git folder already deleted") try: shutil.rmtree(os.path.join(_config["DOCS_FOLDER"], fork_proj_location)) except: print("docs folder already deleted") try: shutil.rmtree( os.path.join(_config["TICKETS_FOLDER"], fork_proj_location) ) except: print("tickets folder already deleted") try: shutil.rmtree( os.path.join(_config["REQUESTS_FOLDER"], fork_proj_location) ) except: print("requests folder already deleted") repo = pagure.lib.query.get_authorized_project(session, "test") result = pagure.lib.query.fork_project(session, "foo", repo) if result == 'Repo "test" cloned to "foo/test"': session.commit() def add_content_git_repo(folder, branch="master"): """ Create some content for the specified git repo. """ if not os.path.exists(folder): os.makedirs(folder) brepo = pygit2.init_repository(folder, bare=True) newfolder = tempfile.mkdtemp(prefix="pagure-tests") repo = pygit2.clone_repository(folder, newfolder) # Create a file in that git repo with open(os.path.join(newfolder, "sources"), "w") as stream: stream.write("foo\n bar") repo.index.add("sources") repo.index.write() parents = [] commit = None try: commit = repo.revparse_single("HEAD" if branch == "master" else branch) except KeyError: pass if commit: parents = [commit.oid.hex] # Commits the files added tree = repo.index.write_tree() author = pygit2.Signature("Alice Author", "alice@authors.tld") committer = pygit2.Signature("Cecil Committer", "cecil@committers.tld") repo.create_commit( "refs/heads/%s" % branch, # the name of the reference to update author, committer, "Add sources file for testing", # binary string representing the tree object ID tree, # list of binary strings representing parents of the new commit parents, ) parents = [] commit = None try: commit = repo.revparse_single("HEAD" if branch == "master" else branch) except KeyError: pass if commit: parents = [commit.oid.hex] subfolder = os.path.join("folder1", "folder2") if not os.path.exists(os.path.join(newfolder, subfolder)): os.makedirs(os.path.join(newfolder, subfolder)) # Create a file in that git repo with open(os.path.join(newfolder, subfolder, "file"), "w") as stream: stream.write("foo\n bar\nbaz") repo.index.add(os.path.join(subfolder, "file")) repo.index.write() # Commits the files added tree = repo.index.write_tree() author = pygit2.Signature("Alice Author", "alice@authors.tld") committer = pygit2.Signature("Cecil Committer", "cecil@committers.tld") repo.create_commit( "refs/heads/%s" % branch, # the name of the reference to update author, committer, "Add some directory and a file for more testing", # binary string representing the tree object ID tree, # list of binary strings representing parents of the new commit parents, ) # Push to origin ori_remote = repo.remotes[0] master_ref = repo.lookup_reference( "HEAD" if branch == "master" else "refs/heads/%s" % branch ).resolve() refname = "%s:%s" % (master_ref.name, master_ref.name) PagureRepo.push(ori_remote, refname) shutil.rmtree(newfolder) def _get_username(): invalid_option = ["pingou", "foo"] user_name = os.environ.get("USER_NAME") if not user_name: print("") user_name = six.moves.input( "Enter your username so we can add you into the test data: " ) cnt = 0 while not user_name.strip() or user_name in invalid_option: print("Reserved names: " + str(invalid_option)) user_name = six.moves.input( "Enter your username so we can add you into the " "test data: " ) cnt += 1 if cnt == 4: print("We asked too many times, bailing") sys.exit(1) return user_name def _get_user_email(): invalid_option = ["bar@pingou.com", "foo@bar.com"] user_email = os.environ.get("USER_EMAIL") if not user_email: print("") user_email = six.moves.input("Enter your user email: ") cnt = 0 while not user_email.strip() or user_email in invalid_option: print("Reserved names: " + str(invalid_option)) user_email = six.moves.input("Enter your user email: ") cnt += 1 if cnt == 4: print("We asked too many times, bailing") sys.exit(1) return user_email if __name__ == "__main__": desc = ( "Run the dev database initialization/insertion/deletion " "script for db located " + str(_config["DB_URL"]) ) parser = argparse.ArgumentParser(prog="dev-data", description=desc) parser.add_argument( "-i", "--init", action="store_true", help="Create the dev db" ) parser.add_argument( "-p", "--populate", action="store_true", help="Add test data to the db" ) parser.add_argument( "-d", "--delete", action="store_true", help="Wipe the dev db" ) parser.add_argument( "-a", "--all", action="store_true", help="Create, Populate then Wipe the dev db", ) args = parser.parse_args() # forcing the user to choose if not any(vars(args).values()): parser.error("No arguments provided.") session = None if args.init or args.all: session = pagure.lib.model.create_tables( db_url=_config["DB_URL"], alembic_ini=None, acls=_config["ACLS"], debug=False, ) print("Database created") if args.populate or args.all: if not session: session = pagure.lib.query.create_session(_config["DB_URL"]) user_name = _get_username() user_email = _get_user_email() insert_data(session, user_name, user_email) if args.delete or args.all: empty_dev_db(session)