123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892 |
- # -*- coding: utf-8 -*-
- """
- (c) 2015-2016 - Copyright Red Hat Inc
- Authors:
- Pierre-Yves Chibon <pingou@pingoured.fr>
- """
- from __future__ import print_function, unicode_literals
- # pylint: disable=too-many-branches
- # pylint: disable=too-many-arguments
- # pylint: disable=too-many-locals
- # pylint: disable=too-many-statements
- # pylint: disable=too-many-lines
- import datetime
- import json
- import logging
- import os
- import shutil
- import subprocess
- import requests
- import tempfile
- import tarfile
- import zipfile
- import arrow
- import pygit2
- import six
- from sqlalchemy.exc import SQLAlchemyError
- # from sqlalchemy.orm.session import Session
- from pygit2.remote import RemoteCollection
- import pagure.utils
- import pagure.exceptions
- import pagure.lib.query
- import pagure.lib.notify
- from pagure.config import config as pagure_config
- from pagure.lib import model
- from pagure.lib.repo import PagureRepo
- from pagure.lib import tasks
- import pagure.hooks
- # from pagure.hooks import run_project_hooks
- _log = logging.getLogger(__name__)
- def commit_to_patch(
- repo_obj, commits, diff_view=False, find_similar=False, separated=False
- ):
- """ For a given commit (PyGit2 commit object) of a specified git repo,
- returns a string representation of the changes the commit did in a
- format that allows it to be used as patch.
- :arg repo_obj: the `pygit2.Repository` object of the git repo to
- retrieve the commits in
- :type repo_obj: `pygit2.Repository`
- :arg commits: the list of commits to convert to path
- :type commits: str or list
- :kwarg diff_view: a boolean specifying if what is returned is a git
- patch or a git diff
- :type diff_view: boolean
- :kwarg find_similar: a boolean specifying if what we run find_similar
- on the diff to group renamed files
- :type find_similar: boolean
- :kwarg separated: a boolean specifying if the data returned should be
- returned as one text blob or not. If diff_view is True, then the diff
- are also split by file, otherwise, the different patches are returned
- as different text blob.
- :type separated: boolean
- :return: the patch or diff representation of the provided commits
- :rtype: str
- """
- if not isinstance(commits, list):
- commits = [commits]
- patch = []
- for cnt, commit in enumerate(commits):
- if commit.parents:
- diff = repo_obj.diff(commit.parents[0], commit)
- else:
- # First commit in the repo
- diff = commit.tree.diff_to_tree(swap=True)
- if find_similar and diff:
- diff.find_similar()
- if diff_view:
- if separated:
- for el in diff.patch.split("\ndiff --git a/"):
- if el and not el.startswith("diff --git a/"):
- patch.append("\ndiff --git a/" + el)
- elif el:
- patch.append(el)
- else:
- patch.append(diff.patch)
- else:
- subject = message = ""
- if "\n" in commit.message:
- subject, message = commit.message.split("\n", 1)
- else:
- subject = commit.message
- if len(commits) > 1:
- subject = "[PATCH %s/%s] %s" % (cnt + 1, len(commits), subject)
- patch.append(
- """From {commit} Mon Sep 17 00:00:00 2001
- From: {author_name} <{author_email}>
- Date: {date}
- Subject: {subject}
- {msg}
- ---
- {patch}
- """.format(
- commit=commit.oid.hex,
- author_name=commit.author.name,
- author_email=commit.author.email,
- date=datetime.datetime.utcfromtimestamp(
- commit.commit_time
- ).strftime("%b %d %Y %H:%M:%S +0000"),
- subject=subject,
- msg=message,
- patch=diff.patch,
- )
- )
- if separated:
- return patch
- else:
- return "".join(patch)
- def generate_gitolite_acls(project=None, group=None):
- """ Generate the gitolite configuration file.
- :arg project: the project of which to update the ACLs. This argument
- can take three values: ``-1``, ``None`` and a project.
- If project is ``-1``, the configuration should be refreshed for
- *all* projects.
- If project is ``None``, there no specific project to refresh
- but the ssh key of an user was added and updated.
- If project is a pagure.lib.model.Project, the configuration of
- this project should be updated.
- :type project: None, int or pagure.lib.model.Project
- :kwarg group: the group to refresh the members of
- :type group: None or str
- """
- if project != -1:
- task = tasks.generate_gitolite_acls.delay(
- namespace=project.namespace if project else None,
- name=project.name if project else None,
- user=project.user.user if project and project.is_fork else None,
- group=group,
- )
- else:
- task = tasks.generate_gitolite_acls.delay(name=-1, group=group)
- return task
- def update_git(obj, repo):
- """ Schedules an update_repo task after determining arguments. """
- ticketuid = None
- requestuid = None
- if obj.isa == "issue":
- ticketuid = obj.uid
- elif obj.isa == "pull-request":
- requestuid = obj.uid
- else:
- raise NotImplementedError("Unknown object type %s" % obj.isa)
- queued = pagure.lib.tasks.update_git.delay(
- repo.name,
- repo.namespace,
- repo.user.username if repo.is_fork else None,
- ticketuid,
- requestuid,
- )
- _maybe_wait(queued)
- return queued
- def _maybe_wait(result):
- """ Function to patch if one wants to wait for finish.
- This function should only ever be overridden by a few tests that depend
- on counting and very precise timing. """
- pass
- def _make_signature(name, email):
- if six.PY2:
- if isinstance(name, six.text_type):
- name = name.encode("utf-8")
- if isinstance(email, six.text_type):
- email = email.encode("utf-8")
- return pygit2.Signature(name=name, email=email)
- def _update_git(obj, repo):
- """ Update the given issue in its git.
- This method forks the provided repo, add/edit the issue whose file name
- is defined by the uid field of the issue and if there are additions/
- changes commit them and push them back to the original repo.
- """
- _log.info("Update the git repo: %s for: %s", repo.path, obj)
- with TemporaryClone(repo, obj.repotype, "update_git") as tempclone:
- if tempclone is None:
- # Turns out we don't have a repo for this kind of object.
- return
- newpath = tempclone.repopath
- new_repo = tempclone.repo
- file_path = os.path.join(newpath, obj.uid)
- # Get the current index
- index = new_repo.index
- # Are we adding files
- added = False
- if not os.path.exists(file_path):
- added = True
- # Write down what changed
- with open(file_path, "w") as stream:
- stream.write(
- json.dumps(
- obj.to_json(),
- sort_keys=True,
- indent=4,
- separators=(",", ": "),
- )
- )
- # Retrieve the list of files that changed
- diff = new_repo.diff()
- files = []
- for patch in diff:
- files.append(patch.delta.new_file.path)
- # Add the changes to the index
- if added:
- index.add(obj.uid)
- for filename in files:
- index.add(filename)
- # If not change, return
- if not files and not added:
- return
- # See if there is a parent to this commit
- parent = None
- try:
- parent = new_repo.head.get_object().oid
- except pygit2.GitError:
- pass
- parents = []
- if parent:
- parents.append(parent)
- # Author/commiter will always be this one
- author = _make_signature(name="pagure", email="pagure")
- # Actually commit
- new_repo.create_commit(
- "refs/heads/master",
- author,
- author,
- "Updated %s %s: %s" % (obj.isa, obj.uid, obj.title),
- new_repo.index.write_tree(),
- parents,
- )
- index.write()
- # And push it back
- tempclone.push("pagure", "master", internal="yes")
- def clean_git(repo, obj_repotype, obj_uid):
- if repo is None:
- return
- task = pagure.lib.tasks.clean_git.delay(
- repo.name,
- repo.namespace,
- repo.user.username if repo.is_fork else None,
- obj_repotype,
- obj_uid,
- )
- _maybe_wait(task)
- return task
- def _clean_git(repo, obj_repotype, obj_uid):
- """ Update the given issue remove it from its git.
- """
- _log.info("Update the git repo: %s to remove: %s", repo.path, obj_uid)
- with TemporaryClone(repo, obj_repotype, "clean_git") as tempclone:
- if tempclone is None:
- # This repo is not tracked on disk
- return
- newpath = tempclone.repopath
- new_repo = tempclone.repo
- file_path = os.path.join(newpath, obj_uid)
- # Get the current index
- index = new_repo.index
- # Are we adding files
- if not os.path.exists(file_path):
- return
- # Remove the file
- os.unlink(file_path)
- # Add the changes to the index
- index.remove(obj_uid)
- # See if there is a parent to this commit
- parent = None
- if not new_repo.is_empty:
- parent = new_repo.head.get_object().oid
- parents = []
- if parent:
- parents.append(parent)
- # Author/commiter will always be this one
- author = _make_signature(name="pagure", email="pagure")
- # Actually commit
- new_repo.create_commit(
- "refs/heads/master",
- author,
- author,
- "Removed object %s: %s" % (obj_repotype, obj_uid),
- new_repo.index.write_tree(),
- parents,
- )
- index.write()
- master_ref = new_repo.lookup_reference("HEAD").resolve().name
- tempclone.push("pagure", master_ref, internal="yes")
- def get_user_from_json(session, jsondata, key="user"):
- """ From the given json blob, retrieve the user info and search for it
- in the db and create the user if it does not already exist.
- """
- user = None
- username = fullname = useremails = default_email = None
- data = jsondata.get(key, None)
- if data:
- username = data.get("name")
- fullname = data.get("fullname")
- useremails = data.get("emails")
- default_email = data.get("default_email")
- if not default_email and useremails:
- default_email = useremails[0]
- if not username and not useremails:
- return
- user = pagure.lib.query.search_user(session, username=username)
- if not user:
- for email in useremails:
- user = pagure.lib.query.search_user(session, email=email)
- if user:
- break
- if not user:
- user = pagure.lib.query.set_up_user(
- session=session,
- username=username,
- fullname=fullname or username,
- default_email=default_email,
- emails=useremails,
- keydir=pagure_config.get("GITOLITE_KEYDIR", None),
- )
- session.commit()
- return user
- def get_project_from_json(session, jsondata):
- """ From the given json blob, retrieve the project info and search for
- it in the db and create the projec if it does not already exist.
- """
- project = None
- user = get_user_from_json(session, jsondata)
- name = jsondata.get("name")
- namespace = jsondata.get("namespace")
- project_user = None
- if jsondata.get("parent"):
- project_user = user.username
- project = pagure.lib.query._get_project(
- session, name, user=project_user, namespace=namespace
- )
- if not project:
- parent = None
- if jsondata.get("parent"):
- parent = get_project_from_json(session, jsondata.get("parent"))
- pagure.lib.query.fork_project(
- session=session, repo=parent, user=user.username
- )
- else:
- pagure.lib.query.new_project(
- session,
- user=user.username,
- name=name,
- namespace=namespace,
- repospanner_region=None,
- description=jsondata.get("description"),
- parent_id=parent.id if parent else None,
- blacklist=pagure_config.get("BLACKLISTED_PROJECTS", []),
- allowed_prefix=pagure_config.get("ALLOWED_PREFIX", []),
- prevent_40_chars=pagure_config.get(
- "OLD_VIEW_COMMIT_ENABLED", False
- ),
- )
- session.commit()
- project = pagure.lib.query._get_project(
- session, name, user=user.username, namespace=namespace
- )
- tags = jsondata.get("tags", None)
- if tags:
- pagure.lib.query.add_tag_obj(
- session, project, tags=tags, user=user.username
- )
- return project
- def update_custom_field_from_json(session, repo, issue, json_data):
- """ Update the custom fields according to the custom fields of
- the issue. If the custom field is not present for the repo in
- it's settings, this will create them.
- :arg session: the session to connect to the database with.
- :arg repo: the sqlalchemy object of the project
- :arg issue: the sqlalchemy object of the issue
- :arg json_data: the json representation of the issue taken from the git
- and used to update the data in the database.
- """
- # Update custom key value, if present
- custom_fields = json_data.get("custom_fields")
- if not custom_fields:
- return
- current_keys = []
- for key in repo.issue_keys:
- current_keys.append(key.name)
- for new_key in custom_fields:
- if new_key["name"] not in current_keys:
- issuekey = model.IssueKeys(
- project_id=repo.id,
- name=new_key["name"],
- key_type=new_key["key_type"],
- )
- try:
- session.add(issuekey)
- session.commit()
- except SQLAlchemyError:
- session.rollback()
- continue
- # The key should be present in the database now
- key_obj = pagure.lib.query.get_custom_key(
- session, repo, new_key["name"]
- )
- value = new_key.get("value")
- if value:
- value = value.strip()
- pagure.lib.query.set_custom_key_value(
- session, issue=issue, key=key_obj, value=value
- )
- try:
- session.commit()
- except SQLAlchemyError:
- session.rollback()
- def update_ticket_from_git(
- session, reponame, namespace, username, issue_uid, json_data, agent
- ):
- """ Update the specified issue (identified by its unique identifier)
- with the data present in the json blob provided.
- :arg session: the session to connect to the database with.
- :arg repo: the name of the project to update
- :arg namespace: the namespace of the project to update
- :arg username: the username of the project to update (if the project
- is a fork)
- :arg issue_uid: the unique identifier of the issue to update
- :arg json_data: the json representation of the issue taken from the git
- and used to update the data in the database.
- :arg agent: the username of the person who pushed the changes (and thus
- is assumed did the action).
- """
- repo = pagure.lib.query._get_project(
- session, reponame, user=username, namespace=namespace
- )
- if not repo:
- raise pagure.exceptions.PagureException(
- "Unknown repo %s of username: %s in namespace: %s"
- % (reponame, username, namespace)
- )
- user = get_user_from_json(session, json_data)
- # rely on the agent provided, but if something goes wrong, behave as
- # ticket creator
- agent = pagure.lib.query.search_user(session, username=agent) or user
- issue = pagure.lib.query.get_issue_by_uid(session, issue_uid=issue_uid)
- messages = []
- if not issue:
- # Create new issue
- pagure.lib.query.new_issue(
- session,
- repo=repo,
- title=json_data.get("title"),
- content=json_data.get("content"),
- priority=json_data.get("priority"),
- user=user.username,
- issue_id=json_data.get("id"),
- issue_uid=issue_uid,
- private=json_data.get("private"),
- status=json_data.get("status"),
- close_status=json_data.get("close_status"),
- date_created=datetime.datetime.utcfromtimestamp(
- float(json_data.get("date_created"))
- ),
- notify=False,
- )
- else:
- # Edit existing issue
- msgs = pagure.lib.query.edit_issue(
- session,
- issue=issue,
- user=agent.username,
- title=json_data.get("title"),
- content=json_data.get("content"),
- priority=json_data.get("priority"),
- status=json_data.get("status"),
- close_status=json_data.get("close_status"),
- private=json_data.get("private"),
- )
- if msgs:
- messages.extend(msgs)
- session.commit()
- issue = pagure.lib.query.get_issue_by_uid(session, issue_uid=issue_uid)
- update_custom_field_from_json(
- session, repo=repo, issue=issue, json_data=json_data
- )
- # Update milestone
- milestone = json_data.get("milestone")
- # If milestone is not in the repo settings, add it
- if milestone:
- if milestone.strip() not in repo.milestones:
- try:
- tmp_milestone = repo.milestones.copy()
- tmp_milestone[milestone.strip()] = None
- repo.milestones = tmp_milestone
- session.add(repo)
- session.commit()
- except SQLAlchemyError:
- session.rollback()
- try:
- msgs = pagure.lib.query.edit_issue(
- session,
- issue=issue,
- user=agent.username,
- milestone=milestone,
- title=json_data.get("title"),
- content=json_data.get("content"),
- status=json_data.get("status"),
- close_status=json_data.get("close_status"),
- private=json_data.get("private"),
- )
- if msgs:
- messages.extend(msgs)
- except SQLAlchemyError:
- session.rollback()
- # Update close_status
- close_status = json_data.get("close_status")
- if close_status:
- if close_status.strip() not in repo.close_status:
- try:
- repo.close_status.append(close_status.strip())
- session.add(repo)
- session.commit()
- except SQLAlchemyError:
- session.rollback()
- # Update tags
- tags = json_data.get("tags", [])
- msgs = pagure.lib.query.update_tags(
- session, issue, tags, username=user.user
- )
- if msgs:
- messages.extend(msgs)
- # Update assignee
- assignee = get_user_from_json(session, json_data, key="assignee")
- if assignee:
- msg = pagure.lib.query.add_issue_assignee(
- session, issue, assignee.username, user=agent.user, notify=False
- )
- if msg:
- messages.append(msg)
- # Update depends
- depends = json_data.get("depends", [])
- msgs = pagure.lib.query.update_dependency_issue(
- session, issue.project, issue, depends, username=agent.user
- )
- if msgs:
- messages.extend(msgs)
- # Update blocks
- blocks = json_data.get("blocks", [])
- msgs = pagure.lib.query.update_blocked_issue(
- session, issue.project, issue, blocks, username=agent.user
- )
- if msgs:
- messages.extend(msgs)
- for comment in json_data["comments"]:
- usercomment = get_user_from_json(session, comment)
- commentobj = pagure.lib.query.get_issue_comment_by_user_and_comment(
- session, issue_uid, usercomment.id, comment["comment"]
- )
- if not commentobj:
- pagure.lib.query.add_issue_comment(
- session,
- issue=issue,
- comment=comment["comment"],
- user=usercomment.username,
- notify=False,
- date_created=datetime.datetime.fromtimestamp(
- float(comment["date_created"])
- ),
- )
- if messages:
- pagure.lib.query.add_metadata_update_notif(
- session=session, obj=issue, messages=messages, user=agent.username
- )
- session.commit()
- def update_request_from_git(
- session, reponame, namespace, username, request_uid, json_data
- ):
- """ Update the specified request (identified by its unique identifier)
- with the data present in the json blob provided.
- :arg session: the session to connect to the database with.
- :arg repo: the name of the project to update
- :arg username: the username to find the repo, is not None for forked
- projects
- :arg request_uid: the unique identifier of the issue to update
- :arg json_data: the json representation of the issue taken from the git
- and used to update the data in the database.
- """
- repo = pagure.lib.query._get_project(
- session, reponame, user=username, namespace=namespace
- )
- if not repo:
- raise pagure.exceptions.PagureException(
- "Unknown repo %s of username: %s in namespace: %s"
- % (reponame, username, namespace)
- )
- user = get_user_from_json(session, json_data)
- request = pagure.lib.query.get_request_by_uid(
- session, request_uid=request_uid
- )
- if not request:
- repo_from = get_project_from_json(session, json_data.get("repo_from"))
- repo_to = get_project_from_json(session, json_data.get("project"))
- status = json_data.get("status")
- if pagure.utils.is_true(status):
- status = "Open"
- elif pagure.utils.is_true(status, ["false"]):
- status = "Merged"
- # Create new request
- pagure.lib.query.new_pull_request(
- session,
- repo_from=repo_from,
- branch_from=json_data.get("branch_from"),
- repo_to=repo_to if repo_to else None,
- remote_git=json_data.get("remote_git"),
- branch_to=json_data.get("branch"),
- title=json_data.get("title"),
- user=user.username,
- requestuid=json_data.get("uid"),
- requestid=json_data.get("id"),
- status=status,
- notify=False,
- )
- session.commit()
- request = pagure.lib.query.get_request_by_uid(
- session, request_uid=request_uid
- )
- # Update start and stop commits
- request.commit_start = json_data.get("commit_start")
- request.commit_stop = json_data.get("commit_stop")
- # Update assignee
- assignee = get_user_from_json(session, json_data, key="assignee")
- if assignee:
- pagure.lib.query.add_pull_request_assignee(
- session, request, assignee.username, user=user.user
- )
- for comment in json_data["comments"]:
- user = get_user_from_json(session, comment)
- commentobj = pagure.lib.query.get_request_comment(
- session, request_uid, comment["id"]
- )
- if not commentobj:
- pagure.lib.query.add_pull_request_comment(
- session,
- request,
- commit=comment["commit"],
- tree_id=comment.get("tree_id") or None,
- filename=comment["filename"],
- row=comment["line"],
- comment=comment["comment"],
- user=user.username,
- notify=False,
- )
- session.commit()
- def _add_file_to_git(repo, issue, attachmentfolder, user, filename):
- """ Add a given file to the specified ticket git repository.
- :arg repo: the Project object from the database
- :arg attachmentfolder: the folder on the filesystem where the attachments
- are stored
- :arg ticketfolder: the folder on the filesystem where the git repo for
- tickets are stored
- :arg user: the user object with its username and email
- :arg filename: the name of the file to save
- """
- with TemporaryClone(repo, issue.repotype, "add_file_to_git") as tempclone:
- newpath = tempclone.repopath
- new_repo = tempclone.repo
- folder_path = os.path.join(newpath, "files")
- file_path = os.path.join(folder_path, filename)
- # Get the current index
- index = new_repo.index
- # Are we adding files
- if os.path.exists(file_path):
- # File exists, remove the clone and return
- shutil.rmtree(newpath)
- return os.path.join("files", filename)
- if not os.path.exists(folder_path):
- os.mkdir(folder_path)
- # Copy from attachments directory
- src = os.path.join(attachmentfolder, repo.fullname, "files", filename)
- shutil.copyfile(src, file_path)
- # Retrieve the list of files that changed
- diff = new_repo.diff()
- files = [patch.new_file_path for patch in diff]
- # Add the changes to the index
- index.add(os.path.join("files", filename))
- for filename in files:
- index.add(filename)
- # See if there is a parent to this commit
- parent = None
- try:
- parent = new_repo.head.get_object().oid
- except pygit2.GitError:
- pass
- parents = []
- if parent:
- parents.append(parent)
- # Author/commiter will always be this one
- author = _make_signature(name=user.username, email=user.default_email)
- # Actually commit
- new_repo.create_commit(
- "refs/heads/master",
- author,
- author,
- "Add file %s to ticket %s: %s"
- % (filename, issue.uid, issue.title),
- new_repo.index.write_tree(),
- parents,
- )
- index.write()
- master_ref = new_repo.lookup_reference("HEAD").resolve()
- tempclone.push(user.username, master_ref.name)
- return os.path.join("files", filename)
- class TemporaryClone(object):
- _project = None
- _action = None
- _repotype = None
- _origpath = None
- _origrepopath = None
- repopath = None
- repo = None
- def __init__(self, project, repotype, action, path=None, parent=None):
- """ Initializes a TempoaryClone instance.
- Args:
- project (model.Project): A project instance
- repotype (string): The type of repo to clone, one of:
- main, docs, requests, tickets
- action (string): Type of action performing, used in the
- temporary directory name
- path (string or None): the path to clone, allows cloning, for
- example remote git repo for remote PRs instead of the
- default one
- parent (string or None): Adds this directory to the path in
- which the project is cloned
- """
- if repotype not in pagure.lib.query.get_repotypes():
- raise NotImplementedError("Repotype %s not known" % repotype)
- self._project = project
- self._repotype = repotype
- self._action = action
- self._path = path
- self._parent = parent
- def __enter__(self):
- """ Enter the context manager, creating the clone. """
- self.repopath = tempfile.mkdtemp(prefix="pagure-%s-" % self._action)
- self._origrepopath = self.repopath
- if self._parent:
- self.repopath = os.path.join(self.repopath, self._parent)
- os.makedirs(self.repopath)
- if not self._project.is_on_repospanner:
- # This is the simple case. Just do a local clone
- # use either the specified path or the use the path of the
- # specified project
- self._origpath = self._path or self._project.repopath(
- self._repotype
- )
- if self._origpath is None:
- # No repository of this type
- # 'main' is already caught and returns an error in repopath()
- return None
- if not os.path.exists(self._origpath):
- return None
- pygit2.clone_repository(self._origpath, self.repopath)
- # Because for whatever reason, one pygit2.Repository is not
- # equal to another.... The pygit2.Repository returned from
- # pygit2.clone_repository does not have the "branches" attribute.
- self.repo = pygit2.Repository(self.repopath)
- self._origrepo = pygit2.Repository(self._origpath)
- else:
- repourl, regioninfo = self._project.repospanner_repo_info(
- self._repotype
- )
- command = [
- "git",
- "-c",
- "protocol.ext.allow=always",
- "clone",
- "ext::%s %s"
- % (
- pagure_config["REPOBRIDGE_BINARY"],
- self._project._repospanner_repo_name(self._repotype),
- ),
- self.repopath,
- ]
- environ = os.environ.copy()
- environ.update(
- {
- "USER": "pagure",
- "REPOBRIDGE_CONFIG": ":environment:",
- "REPOBRIDGE_BASEURL": regioninfo["url"],
- "REPOBRIDGE_CA": regioninfo["ca"],
- "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"],
- "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"],
- }
- )
- with open(os.devnull, "w") as devnull:
- subprocess.check_call(
- command,
- stdout=devnull,
- stderr=subprocess.STDOUT,
- env=environ,
- )
- self.repo = pygit2.Repository(self.repopath)
- # Make sure that all remote refs are mapped to local ones.
- headname = None
- if not self.repo.is_empty and not self.repo.head_is_unborn:
- headname = self.repo.head.shorthand
- # Sync up all the references, branches and PR heads
- for ref in self._origrepo.listall_references():
- if ref.startswith("refs/heads/"):
- localname = ref.replace("refs/heads/", "")
- if localname in (headname, "HEAD"):
- # This gets checked out by default
- continue
- branch = self.repo.branches.remote.get("origin/%s" % localname)
- self.repo.branches.local.create(localname, branch.get_object())
- elif ref.startswith("refs/pull/"):
- reference = self._origrepo.references.get(ref)
- self.repo.references.create(
- ref, reference.get_object().oid.hex
- )
- return self
- def __exit__(self, exc_type, exc_value, traceback):
- """ Exit the context manager, removing the temorary clone. """
- shutil.rmtree(self.repopath)
- def push(self, username, sbranch, tbranch=None, force=False, **extra):
- """ Push the repo back to its origin.
- Args:
- username (string): The user on who's account this push is
- sbranch (string): Source branch to push
- tbranch (string): Target branch if different from sbranch
- extra (dict): Extra fields passed to the remote side. Either via
- environment variables, or as X-Extra-<key> HTTP headers.
- """
- pushref = "%s:%s" % (sbranch, tbranch if tbranch else sbranch)
- if "pull_request" in extra:
- extra["pull_request_uid"] = extra["pull_request"].uid
- del extra["pull_request"]
- if self._project.is_on_repospanner:
- regioninfo = pagure_config["REPOSPANNER_REGIONS"][
- self._project.repospanner_region
- ]
- extra.update(
- {
- "username": username,
- "repotype": self._repotype,
- "project_name": self._project.name,
- "project_user": self._project.user.username
- if self._project.is_fork
- else "",
- "project_namespace": self._project.namespace or "",
- }
- )
- args = []
- for opt in extra:
- args.extend(["--extra", opt, extra[opt]])
- command = [
- "git",
- "-c",
- "protocol.ext.allow=always",
- "push",
- "ext::%s %s %s"
- % (
- pagure_config["REPOBRIDGE_BINARY"],
- " ".join(args),
- self._project._repospanner_repo_name(self._repotype),
- ),
- "--repo",
- self.repopath,
- ]
- environ = {
- "USER": "pagure",
- "REPOBRIDGE_CONFIG": ":environment:",
- "REPOBRIDGE_BASEURL": regioninfo["url"],
- "REPOBRIDGE_CA": regioninfo["ca"],
- "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"],
- "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"],
- }
- else:
- command = ["git", "push", "origin"]
- if force:
- command.append("--force")
- environ = {}
- try:
- _log.debug(
- "Running a git push of %s to %s"
- % (pushref, self._path or self._project.fullname)
- )
- env = os.environ.copy()
- env["GL_USER"] = username
- env["GL_BYPASS_ACCESS_CHECKS"] = "1"
- if pagure_config.get("GITOLITE_HOME"):
- env["HOME"] = pagure_config["GITOLITE_HOME"]
- env.update(environ)
- env.update(extra)
- out = subprocess.check_output(
- command + [pushref],
- cwd=self.repopath,
- stderr=subprocess.STDOUT,
- env=env,
- )
- _log.debug("Output: %s" % out)
- except subprocess.CalledProcessError as ex:
- # This should never really happen, since we control the repos, but
- # this way, we can be sure to get the output logged
- remotes = []
- for line in ex.output.decode("utf-8").split("\n"):
- _log.info("Remote line: %s", line)
- if line.startswith("remote: "):
- _log.debug("Remote: %s" % line)
- remotes.append(line[len("remote: ") :].strip())
- if remotes:
- _log.info("Remote rejected with: %s" % remotes)
- raise pagure.exceptions.PagurePushDenied(
- "Remote hook declined the push: %s" % "\n".join(remotes)
- )
- else:
- # Something else happened, pass the original
- _log.exception("Error pushing. Output: %s", ex.output)
- raise
- def _update_file_in_git(
- repo, branch, branchto, filename, content, message, user, email
- ):
- """ Update a specific file in the specified repository with the content
- given and commit the change under the user's name.
- :arg repo: the Project object from the database
- :arg branch: the branch from which the edit is made
- :arg branchto: the name of the branch into which to edit the file
- :arg filename: the name of the file to save
- :arg content: the new content of the file
- :arg message: the message of the git commit
- :arg user: the user name, to use in the commit
- :arg email: the email of the user, to use in the commit
- """
- _log.info("Updating file: %s in the repo: %s", filename, repo.path)
- with TemporaryClone(repo, "main", "edit_file") as tempclone:
- newpath = tempclone.repopath
- new_repo = tempclone.repo
- new_repo.checkout("refs/heads/%s" % branch)
- file_path = os.path.join(newpath, filename)
- # Get the current index
- index = new_repo.index
- # Write down what changed
- with open(file_path, "wb") as stream:
- stream.write(content.replace("\r", "").encode("utf-8"))
- # Retrieve the list of files that changed
- diff = new_repo.diff()
- files = []
- for patch in diff:
- files.append(patch.delta.new_file.path)
- # Add the changes to the index
- added = False
- for filename in files:
- added = True
- index.add(filename)
- # If not change, return
- if not files and not added:
- return
- # See if there is a parent to this commit
- branch_ref = get_branch_ref(new_repo, branch)
- parent = branch_ref.get_object()
- # See if we need to create the branch
- nbranch_ref = None
- if branchto not in new_repo.listall_branches():
- nbranch_ref = new_repo.create_branch(branchto, parent)
- parents = []
- if parent:
- parents.append(parent.hex)
- # Author/commiter will always be this one
- name = user.fullname or user.username
- author = _make_signature(name=name, email=email)
- # Actually commit
- new_repo.create_commit(
- nbranch_ref.name if nbranch_ref else branch_ref.name,
- author,
- author,
- message.strip(),
- new_repo.index.write_tree(),
- parents,
- )
- index.write()
- tempclone.push(
- user.username,
- nbranch_ref.name if nbranch_ref else branch_ref.name,
- branchto,
- )
- return os.path.join("files", filename)
- def read_output(cmd, abspath, input=None, keepends=False, error=False, **kw):
- """ Read the output from the given command to run.
- cmd:
- The command to run, this is a list with each space separated into an
- element of the list.
- abspath:
- The absolute path where the command should be ran.
- input:
- Whether the command should take input from stdin or not.
- (Defaults to False)
- keepends:
- Whether to strip the newline characters at the end of the standard
- output or not.
- error:
- Whether to return both the standard output and the standard error,
- or just the standard output.
- (Defaults to False).
- kw*:
- Any other arguments to be passed onto the subprocess.Popen command,
- such as env, shell, executable...
- """
- if input:
- stdin = subprocess.PIPE
- else:
- stdin = None
- procs = subprocess.Popen(
- cmd,
- stdin=stdin,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=abspath,
- **kw
- )
- retcode = procs.wait()
- (out, err) = procs.communicate(input)
- if not isinstance(out, str):
- out = out.decode("utf-8")
- if not isinstance(err, str):
- err = err.decode("utf-8")
- if retcode:
- print("ERROR: %s =-- %s" % (cmd, retcode))
- print(out)
- print(err)
- if not keepends:
- out = out.rstrip("\n\r")
- if error:
- return (out, err)
- else:
- return out
- def read_git_output(
- args, abspath, input=None, keepends=False, error=False, **kw
- ):
- """Read the output of a Git command."""
- return read_output(
- ["git"] + args,
- abspath,
- input=input,
- keepends=keepends,
- error=error,
- **kw
- )
- def read_git_lines(args, abspath, keepends=False, error=False, **kw):
- """Return the lines output by Git command.
- Return as single lines, with newlines stripped off."""
- if error:
- return read_git_output(
- args, abspath, keepends=keepends, error=error, **kw
- )
- else:
- return read_git_output(
- args, abspath, keepends=keepends, **kw
- ).splitlines(keepends)
- def get_revs_between(oldrev, newrev, abspath, refname, forced=False):
- """ Yield revisions between HEAD and BASE. """
- cmd = ["rev-list", "%s...%s" % (oldrev, newrev)]
- if forced:
- head = get_default_branch(abspath)
- cmd.append("^%s" % head)
- if set(newrev) == set("0"):
- cmd = ["rev-list", "%s" % oldrev]
- elif set(oldrev) == set("0") or set(oldrev) == set("^0"):
- head = get_default_branch(abspath)
- cmd = ["rev-list", "%s" % newrev, "^%s" % head]
- if head in refname:
- cmd = ["rev-list", "%s" % newrev]
- return pagure.lib.git.read_git_lines(cmd, abspath)
- def is_forced_push(oldrev, newrev, abspath):
- """ Returns whether there was a force push between HEAD and BASE.
- Doc: http://stackoverflow.com/a/12258773
- """
- if set(oldrev) == set("0"):
- # This is a push that's creating a new branch => certainly ok
- return False
- # Returns if there was any commits deleted in the changeset
- cmd = ["rev-list", "%s" % oldrev, "^%s" % newrev]
- out = pagure.lib.git.read_git_lines(cmd, abspath)
- return len(out) > 0
- def get_base_revision(torev, fromrev, abspath):
- """ Return the base revision between HEAD and BASE.
- This is useful in case of force-push.
- """
- cmd = ["merge-base", fromrev, torev]
- return pagure.lib.git.read_git_lines(cmd, abspath)
- def get_default_branch(abspath):
- """ Return the default branch of a repo. """
- cmd = ["rev-parse", "--abbrev-ref", "HEAD"]
- out = pagure.lib.git.read_git_lines(cmd, abspath)
- if out:
- return out[0]
- else:
- return "master"
- def get_author(commit, abspath):
- """ Return the name of the person that authored the commit. """
- user = pagure.lib.git.read_git_lines(
- ["log", "-1", '--pretty=format:"%an"', commit], abspath
- )[0].replace('"', "")
- return user
- def get_author_email(commit, abspath):
- """ Return the email of the person that authored the commit. """
- user = pagure.lib.git.read_git_lines(
- ["log", "-1", '--pretty=format:"%ae"', commit], abspath
- )[0].replace('"', "")
- return user
- def get_commit_subject(commit, abspath):
- """ Return the subject of the commit. """
- subject = pagure.lib.git.read_git_lines(
- ["log", "-1", '--pretty=format:"%s"', commit], abspath
- )[0].replace('"', "")
- return subject
- def get_repo_info_from_path(gitdir, hide_notfound=False):
- """ Returns the name, username, namespace and type of a git directory
- This gets computed based on the *_FOLDER's in the config file,
- and as such only works for the central file-based repositories.
- Args:
- gitdir (string): Path of the canonical git repository
- hide_notfound (bool): Whether to return a tuple with None's instead of
- raising an error if the regenerated repo didn't exist.
- Can be used to hide the difference between no project access vs not
- existing when looking up private repos.
- Return: (tuple): Tuple with (repotype, username, namespace, repo)
- Some of these elements may be None if not applicable.
- """
- if not os.path.isabs(gitdir):
- raise ValueError("Tried to locate non-absolute gitdir %s" % gitdir)
- gitdir = os.path.normpath(gitdir)
- types = {
- "main": pagure_config["GIT_FOLDER"],
- "docs": pagure_config["DOCS_FOLDER"],
- "tickets": pagure_config["TICKETS_FOLDER"],
- "requests": pagure_config["REQUESTS_FOLDER"],
- }
- match = None
- matchlen = None
- # First find the longest match in types. This makes sure that even if the
- # non-main repos are in a subdir of main (i.e. repos/ and repos/tickets/),
- # we find the correct type.
- for typename in types:
- if not types[typename]:
- continue
- types[typename] = os.path.abspath(types[typename])
- path = types[typename] + "/"
- if gitdir.startswith(path) and (
- matchlen is None or len(path) > matchlen
- ):
- match = typename
- matchlen = len(path)
- if match is None:
- raise ValueError("Gitdir %s could not be located" % gitdir)
- typepath = types[match]
- guesspath = gitdir[len(typepath) + 1 :]
- if len(guesspath) < 5:
- # At least 4 characters for ".git" is required plus one for project
- # name
- raise ValueError("Invalid gitdir %s located" % gitdir)
- # Just in the case we run on a non-*nix system...
- guesspath = guesspath.replace("\\", "/")
- # Now guesspath should be one of:
- # - reponame.git
- # - namespace/reponame.git
- # - forks/username/reponame.git
- # - forks/username/namespace/reponame.git
- repotype = match
- username = None
- namespace = None
- repo = None
- guesspath, repo = os.path.split(guesspath)
- if not repo.endswith(".git"):
- raise ValueError("Git dir looks to not be a bare repo")
- repo = repo[: -len(".git")]
- if not repo:
- raise ValueError("Gitdir %s seems to not be a bare repo" % gitdir)
- # Split the guesspath up, throwing out any empty strings
- splitguess = [part for part in guesspath.split("/") if part]
- if splitguess and splitguess[0] == "forks":
- if len(splitguess) < 2:
- raise ValueError("Invalid gitdir %s" % gitdir)
- username = splitguess[1]
- splitguess = splitguess[2:]
- if splitguess:
- # At this point, we've cut off the repo name at the end, and any forks/
- # indicators and their usernames are also removed, so remains just the
- # namespace
- namespace = os.path.join(*splitguess)
- # Okay, we think we have everything. Let's make doubly sure the path is
- # correct and exists
- rebuiltpath = os.path.join(
- typepath,
- "forks/" if username else "",
- username if username else "",
- namespace if namespace else "",
- repo + ".git",
- )
- if os.path.normpath(rebuiltpath) != gitdir:
- raise ValueError(
- "Rebuilt %s path not identical to gitdir %s"
- % (rebuiltpath, gitdir)
- )
- if not os.path.exists(rebuiltpath) and not hide_notfound:
- raise ValueError("Splitting gitdir %s failed" % gitdir)
- return (repotype, username, namespace, repo)
- def get_repo_name(abspath):
- """ Return the name of the git repo based on its path.
- """
- _, _, _, name = get_repo_info_from_path(abspath)
- return name
- def get_repo_namespace(abspath, gitfolder=None):
- """ Return the name of the git repo based on its path.
- """
- _, _, namespace, _ = get_repo_info_from_path(abspath)
- return namespace
- def get_username(abspath):
- """ Return the username of the git repo based on its path.
- """
- _, username, _, _ = get_repo_info_from_path(abspath)
- return username
- def get_branch_ref(repo, branchname):
- """ Return the reference to the specified branch or raises an exception.
- """
- location = pygit2.GIT_BRANCH_LOCAL
- if branchname not in repo.listall_branches():
- branchname = "origin/%s" % branchname
- location = pygit2.GIT_BRANCH_REMOTE
- branch_ref = repo.lookup_branch(branchname, location)
- if not branch_ref or not branch_ref.resolve():
- raise pagure.exceptions.PagureException(
- "No refs found for %s" % branchname
- )
- return branch_ref.resolve()
- def merge_pull_request(session, request, username, domerge=True):
- """ Merge the specified pull-request.
- """
- if domerge:
- _log.info("%s asked to merge the pull-request: %s", username, request)
- else:
- _log.info("%s asked to diff the pull-request: %s", username, request)
- repopath = None
- if request.remote:
- # Get the fork
- repopath = pagure.utils.get_remote_repo_path(
- request.remote_git, request.branch_from
- )
- elif request.project_from:
- # Get the fork
- repopath = pagure.utils.get_repo_path(request.project_from)
- fork_obj = None
- if repopath:
- fork_obj = PagureRepo(repopath)
- with TemporaryClone(request.project, "main", "merge_pr") as tempclone:
- new_repo = tempclone.repo
- # Update the start and stop commits in the DB, one last time
- diff_commits = diff_pull_request(
- session, request, fork_obj, new_repo, with_diff=False
- )
- _log.info(" %s commit to merge", len(diff_commits))
- if request.project.settings.get(
- "Enforce_signed-off_commits_in_pull-request", False
- ):
- for commit in diff_commits:
- if "signed-off-by" not in commit.message.lower():
- _log.info(" Missing a required: signed-off-by: Bailing")
- raise pagure.exceptions.PagureException(
- "This repo enforces that all commits are "
- "signed off by their author. "
- )
- if not new_repo.is_empty and not new_repo.head_is_unborn:
- try:
- branch_ref = get_branch_ref(new_repo, request.branch)
- except pagure.exceptions.PagureException:
- branch_ref = None
- if not branch_ref:
- _log.info(" Target branch could not be found")
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s could not be found in the repo %s"
- % (request.branch, request.project.fullname)
- )
- new_repo.checkout(branch_ref)
- if fork_obj:
- # Check/Get the branch from
- branch = None
- try:
- branch = get_branch_ref(fork_obj, request.branch_from)
- except pagure.exceptions.PagureException:
- pass
- if not branch:
- _log.info(" Branch of origin could not be found")
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s could not be found in the repo %s"
- % (
- request.branch_from,
- request.project_from.fullname
- if request.project_from
- else request.remote_git,
- )
- )
- # Add the fork as remote repo
- reponame = "%s_%s" % (request.user.user, request.uid)
- _log.info(
- " Adding remote: %s pointing to: %s", reponame, repopath
- )
- remote = new_repo.create_remote(reponame, repopath)
- # Fetch the commits
- remote.fetch()
- # repo_commit = fork_obj[branch.get_object().hex]
- repo_commit = new_repo[branch.get_object().hex]
- # Checkout the correct branch
- if new_repo.is_empty or new_repo.head_is_unborn:
- _log.debug(
- " target repo is empty, so PR can be merged using "
- "fast-forward, reporting it"
- )
- if domerge:
- _log.info(" PR merged using fast-forward")
- if not request.project.settings.get("always_merge", False):
- new_repo.create_branch(request.branch, repo_commit)
- commit = repo_commit.oid.hex
- else:
- tree = new_repo.index.write_tree()
- user_obj = pagure.lib.query.get_user(session, username)
- commitname = user_obj.fullname or user_obj.user
- author = _make_signature(
- commitname, user_obj.default_email
- )
- commit = new_repo.create_commit(
- "refs/heads/%s" % request.branch,
- author,
- author,
- "Merge #%s `%s`" % (request.id, request.title),
- tree,
- [repo_commit.oid.hex],
- )
- _log.info(" New head: %s", commit)
- tempclone.push(
- username,
- request.branch,
- request.branch,
- pull_request=request,
- )
- # Update status
- _log.info(" Closing the PR in the DB")
- pagure.lib.query.close_pull_request(
- session, request, username
- )
- return "Changes merged!"
- else:
- _log.info(
- " PR can be merged using fast-forward, reporting it"
- )
- request.merge_status = "FFORWARD"
- session.commit()
- return "FFORWARD"
- else:
- try:
- ref = new_repo.lookup_reference(
- "refs/pull/%s/head" % request.id
- )
- repo_commit = new_repo[ref.target.hex]
- except KeyError:
- pass
- merge = new_repo.merge(repo_commit.oid)
- _log.debug(" Merge: %s", merge)
- if merge is None:
- mergecode = new_repo.merge_analysis(repo_commit.oid)[0]
- _log.debug(" Mergecode: %s", mergecode)
- # Wait until the last minute then check if the PR was already closed
- # by someone else in the mean while and if so, just bail
- if request.status != "Open":
- _log.info(
- " This pull-request has already been merged or closed by %s "
- "on %s" % (request.closed_by.user, request.closed_at)
- )
- raise pagure.exceptions.PagureException(
- "This pull-request was merged or closed by %s"
- % request.closed_by.user
- )
- if (merge is not None and merge.is_uptodate) or ( # noqa
- merge is None and mergecode & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE
- ):
- if domerge:
- _log.info(" PR up to date, closing it")
- pagure.lib.query.close_pull_request(session, request, username)
- try:
- session.commit()
- except SQLAlchemyError: # pragma: no cover
- session.rollback()
- _log.exception(" Could not merge the PR in the DB")
- raise pagure.exceptions.PagureException(
- "Could not close this pull-request"
- )
- raise pagure.exceptions.PagureException(
- "Nothing to do, changes were already merged"
- )
- else:
- _log.info(" PR up to date, reporting it")
- request.merge_status = "NO_CHANGE"
- session.commit()
- return "NO_CHANGE"
- elif (merge is not None and merge.is_fastforward) or ( # noqa
- merge is None and mergecode & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD
- ):
- if domerge:
- _log.info(" PR merged using fast-forward")
- head = new_repo.lookup_reference("HEAD").get_object()
- if not request.project.settings.get("always_merge", False):
- if merge is not None:
- # This is depending on the pygit2 version
- branch_ref.target = merge.fastforward_oid
- elif merge is None and mergecode is not None:
- branch_ref.set_target(repo_commit.oid.hex)
- commit = repo_commit.oid.hex
- else:
- tree = new_repo.index.write_tree()
- user_obj = pagure.lib.query.get_user(session, username)
- commitname = user_obj.fullname or user_obj.user
- author = _make_signature(
- commitname, user_obj.default_email
- )
- commit_message = "Merge #%s `%s`" % (
- request.id,
- request.title,
- )
- if request.project.settings.get(
- "Enforce_signed-off_commits_in_pull-request", False
- ):
- commit_message += "\n\nSigned-off-by: %s <%s>" % (
- commitname,
- user_obj.default_email,
- )
- commit = new_repo.create_commit(
- "refs/heads/%s" % request.branch,
- author,
- author,
- commit_message,
- tree,
- [head.hex, repo_commit.oid.hex],
- )
- _log.info(" New head: %s", commit)
- tempclone.push(
- username,
- branch_ref.name,
- request.branch,
- pull_request=request,
- )
- else:
- _log.info(
- " PR can be merged using fast-forward, reporting it"
- )
- request.merge_status = "FFORWARD"
- session.commit()
- return "FFORWARD"
- else:
- tree = None
- try:
- tree = new_repo.index.write_tree()
- except pygit2.GitError as err:
- _log.debug(
- " Could not write down the new tree: " "merge conflicts"
- )
- _log.debug(err)
- if domerge:
- _log.info(" Merge conflict: Bailing")
- raise pagure.exceptions.PagureException("Merge conflicts!")
- else:
- _log.info(" Merge conflict, reporting it")
- request.merge_status = "CONFLICTS"
- session.commit()
- return "CONFLICTS"
- if domerge:
- if request.project.settings.get(
- "disable_non_fast-forward_merges", False
- ):
- _log.info(" Merge non-FF PR is disabled for this project")
- return "MERGE"
- _log.info(" Writing down merge commit")
- head = new_repo.lookup_reference("HEAD").get_object()
- _log.info(
- " Basing on: %s - %s", head.hex, repo_commit.oid.hex
- )
- user_obj = pagure.lib.query.get_user(session, username)
- commitname = user_obj.fullname or user_obj.user
- author = _make_signature(commitname, user_obj.default_email)
- commit_message = "Merge #%s `%s`" % (request.id, request.title)
- if request.project.settings.get(
- "Enforce_signed-off_commits_in_pull-request", False
- ):
- commit_message += "\n\nSigned-off-by: %s <%s>" % (
- commitname,
- user_obj.default_email,
- )
- commit = new_repo.create_commit(
- "refs/heads/%s" % request.branch,
- author,
- author,
- commit_message,
- tree,
- [head.hex, repo_commit.oid.hex],
- )
- _log.info(" New head: %s", commit)
- local_ref = "refs/heads/_pagure_topush"
- new_repo.create_reference(local_ref, commit)
- tempclone.push(
- username, local_ref, request.branch, pull_request=request
- )
- else:
- _log.info(
- " PR can be merged with a merge commit, " "reporting it"
- )
- request.merge_status = "MERGE"
- session.commit()
- return "MERGE"
- # Update status
- _log.info(" Closing the PR in the DB")
- pagure.lib.query.close_pull_request(session, request, username)
- return "Changes merged!"
- def rebase_pull_request(request, username):
- """ Rebase the specified pull-request.
- Args:
- session (sqlalchemy): the session to connect to the database with
- request (pagure.lib.model.PullRequest): the database object
- corresponding to the pull-request to rebase
- username (string): the name of the user asking for the pull-request
- to be rebased
- Returns: (string or None): Pull-request rebased
- Raises: pagure.exceptions.PagureException
- """
- _log.info("%s asked to rebased the pull-request: %s", username, request)
- if request.remote:
- # Get the fork
- repopath = pagure.utils.get_remote_repo_path(
- request.remote_git, request.branch_from
- )
- elif request.project_from:
- # Get the fork
- repopath = pagure.utils.get_repo_path(request.project_from)
- else:
- _log.info(
- "PR is neither from a remote git repo or an existing local "
- "repo, bailing"
- )
- return
- if not request.project or not os.path.exists(
- pagure.utils.get_repo_path(request.project)
- ):
- _log.info(
- "Could not find the targeted git repository for %s",
- request.project.fullname,
- )
- raise pagure.exceptions.PagureException(
- "Could not find the targeted git repository for %s"
- % request.project.fullname
- )
- with TemporaryClone(
- project=request.project,
- repotype="main",
- action="rebase_pr",
- path=repopath,
- ) as tempclone:
- new_repo = tempclone.repo
- new_repo.checkout("refs/heads/%s" % request.branch_from)
- # Add the upstream repo as remote
- upstream = "%s_%s" % (request.user.user, request.uid)
- upstream_path = pagure.utils.get_repo_path(request.project)
- _log.info(
- " Adding remote: %s pointing to: %s", upstream, upstream_path
- )
- remote = new_repo.create_remote(upstream, upstream_path)
- # Fetch the commits
- remote.fetch()
- def _run_command(command):
- _log.info("Running command: %s", command)
- try:
- out = subprocess.check_output(
- command, cwd=tempclone.repopath, stderr=subprocess.STDOUT
- )
- _log.info(" command ran successfully")
- _log.debug("Output: %s" % out)
- except subprocess.CalledProcessError as err:
- _log.debug(
- "Rebase FAILED: {cmd} returned code {code} with the "
- "following output: {output}".format(
- cmd=err.cmd, code=err.returncode, output=err.output
- )
- )
- raise pagure.exceptions.PagureException(
- "Did not manage to rebase this pull-request"
- )
- # Configure git for that user
- command = ["git", "config", "user.name", username]
- _run_command(command)
- command = ["git", "config", "user.email", "%s@pagure" % username]
- _run_command(command)
- # Do the rebase
- command = ["git", "pull", "--rebase", upstream, request.branch]
- _run_command(command)
- # Retrieve the reference of the branch we're working on
- try:
- branch_ref = get_branch_ref(new_repo, request.branch_from)
- except pagure.exceptions.PagureException:
- branch_ref = None
- if not branch_ref:
- _log.debug(" Target branch could not be found")
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s could not be found in the repo %s"
- % (request.branch, request.project.fullname)
- )
- # Push the changes
- _log.info("Pushing %s to %s", branch_ref.name, request.branch_from)
- try:
- tempclone.push(
- username,
- branch_ref.name,
- request.branch_from,
- pull_request=request,
- force=True,
- )
- except subprocess.CalledProcessError as err:
- _log.debug(
- "Rebase FAILED: {cmd} returned code {code} with the "
- "following output: {output}".format(
- cmd=err.cmd, code=err.returncode, output=err.output
- )
- )
- raise pagure.exceptions.PagureException(
- "Did not manage to rebase this pull-request"
- )
- return "Pull-request rebased"
- def get_diff_info(repo_obj, orig_repo, branch_from, branch_to, prid=None):
- """ Return the info needed to see a diff or make a Pull-Request between
- the two specified repo.
- :arg repo_obj: The pygit2.Repository object of the first git repo
- :arg orig_repo: The pygit2.Repository object of the second git repo
- :arg branch_from: the name of the branch having the changes, in the
- first git repo
- :arg branch_to: the name of the branch in which we want to merge the
- changes in the second git repo
- :kwarg prid: the identifier of the pull-request to
- """
- try:
- frombranch = repo_obj.lookup_branch(branch_from)
- except ValueError:
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s does not exist" % branch_from
- )
- except AttributeError:
- frombranch = None
- if not frombranch and prid is None and repo_obj and not repo_obj.is_empty:
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s does not exist" % branch_from
- )
- branch = None
- if branch_to:
- try:
- branch = orig_repo.lookup_branch(branch_to)
- except ValueError:
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s does not exist" % branch_to
- )
- local_branches = orig_repo.listall_branches(pygit2.GIT_BRANCH_LOCAL)
- if not branch and local_branches:
- raise pagure.exceptions.BranchNotFoundException(
- "Branch %s could not be found in the target repo" % branch_to
- )
- commitid = None
- if frombranch:
- commitid = frombranch.get_object().hex
- elif prid is not None:
- # If there is not branch found but there is a PR open, use the ref
- # of that PR in the main repo
- try:
- ref = orig_repo.lookup_reference("refs/pull/%s/head" % prid)
- commitid = ref.target.hex
- except KeyError:
- pass
- if not commitid and repo_obj and not repo_obj.is_empty:
- raise pagure.exceptions.PagureException(
- "No branch from which to pull or local PR reference were found"
- )
- diff_commits = []
- diff = None
- orig_commit = None
- # If the fork is empty but there is a PR open, use the main repo
- if (not repo_obj or repo_obj.is_empty) and prid is not None:
- repo_obj = orig_repo
- if not repo_obj.is_empty and not orig_repo.is_empty:
- _log.info(
- "pagure.lib.git.get_diff_info: Pulling into a non-empty repo"
- )
- if branch:
- orig_commit = orig_repo[branch.get_object().hex]
- main_walker = orig_repo.walk(
- orig_commit.oid.hex, pygit2.GIT_SORT_NONE
- )
- repo_commit = repo_obj[commitid]
- branch_walker = repo_obj.walk(
- repo_commit.oid.hex, pygit2.GIT_SORT_NONE
- )
- main_commits = set()
- branch_commits = set()
- while 1:
- com = None
- if branch:
- try:
- com = next(main_walker)
- main_commits.add(com.oid.hex)
- except StopIteration:
- com = None
- try:
- branch_commit = next(branch_walker)
- except StopIteration:
- branch_commit = None
- # We sure never end up here but better safe than sorry
- if com is None and branch_commit is None:
- break
- if branch_commit:
- branch_commits.add(branch_commit.oid.hex)
- diff_commits.append(branch_commit)
- if main_commits.intersection(branch_commits):
- break
- # If master is ahead of branch, we need to remove the commits
- # that are after the first one found in master
- i = 0
- if diff_commits and main_commits:
- for i in range(len(diff_commits)):
- if diff_commits[i].oid.hex in main_commits:
- break
- diff_commits = diff_commits[:i]
- _log.debug("Diff commits: %s", diff_commits)
- if diff_commits:
- first_commit = repo_obj[diff_commits[-1].oid.hex]
- if len(first_commit.parents) > 0:
- diff = repo_obj.diff(
- repo_obj.revparse_single(first_commit.parents[0].oid.hex),
- repo_obj.revparse_single(diff_commits[0].oid.hex),
- )
- elif first_commit.oid.hex == diff_commits[0].oid.hex:
- _log.info(
- "pagure.lib.git.get_diff_info: First commit is also the "
- "last commit"
- )
- diff = diff_commits[0].tree.diff_to_tree(swap=True)
- elif orig_repo.is_empty and repo_obj and not repo_obj.is_empty:
- _log.info("pagure.lib.git.get_diff_info: Pulling into an empty repo")
- if "master" in repo_obj.listall_branches():
- repo_commit = repo_obj[repo_obj.head.target]
- else:
- branch = repo_obj.lookup_branch(branch_from)
- repo_commit = branch.get_object()
- for commit in repo_obj.walk(repo_commit.oid.hex, pygit2.GIT_SORT_NONE):
- diff_commits.append(commit)
- _log.debug("Diff commits: %s", diff_commits)
- diff = repo_commit.tree.diff_to_tree(swap=True)
- else:
- raise pagure.exceptions.PagureException(
- "Fork is empty, there are no commits to create a pull "
- "request with"
- )
- _log.info(
- "pagure.lib.git.get_diff_info: diff_commits length: %s",
- len(diff_commits),
- )
- _log.info("pagure.lib.git.get_diff_info: original commit: %s", orig_commit)
- return (diff, diff_commits, orig_commit)
- def diff_pull_request(
- session, request, repo_obj, orig_repo, with_diff=True, notify=True
- ):
- """ Returns the diff and the list of commits between the two git repos
- mentionned in the given pull-request.
- :arg session: The sqlalchemy session to connect to the database
- :arg request: The pagure.lib.model.PullRequest object of the pull-request
- to look into
- :arg repo_obj: The pygit2.Repository object of the first git repo
- :arg orig_repo: The pygit2.Repository object of the second git repo
- :arg with_diff: A boolean on whether to return the diff with the list
- of commits (or just the list of commits)
- """
- diff = None
- diff_commits = []
- diff, diff_commits, _ = get_diff_info(
- repo_obj,
- orig_repo,
- request.branch_from,
- request.branch,
- prid=request.id,
- )
- if request.status == "Open" and diff_commits:
- first_commit = diff_commits[-1]
- # Check if we can still rely on the merge_status
- commenttext = None
- if (
- request.commit_start != first_commit.oid.hex
- or request.commit_stop != diff_commits[0].oid.hex
- ):
- request.merge_status = None
- if request.commit_start:
- pr_action = "updated"
- new_commits_count = 0
- commenttext = ""
- for i in diff_commits:
- if i.oid.hex == request.commit_stop:
- break
- new_commits_count = new_commits_count + 1
- commenttext = "%s * ``%s``\n" % (
- commenttext,
- i.message.strip().split("\n")[0],
- )
- if new_commits_count == 1:
- commenttext = "**%d new commit added**\n\n%s" % (
- new_commits_count,
- commenttext,
- )
- else:
- commenttext = "**%d new commits added**\n\n%s" % (
- new_commits_count,
- commenttext,
- )
- if (
- request.commit_start
- and request.commit_start != first_commit.oid.hex
- ):
- pr_action = "rebased"
- commenttext = "rebased onto %s" % first_commit.oid.hex
- request.commit_start = first_commit.oid.hex
- request.commit_stop = diff_commits[0].oid.hex
- session.add(request)
- session.commit()
- tasks.sync_pull_ref.delay(
- request.project.name,
- request.project.namespace,
- request.project.user.username if request.project.is_fork else None,
- request.id,
- )
- if commenttext:
- tasks.link_pr_to_ticket.delay(request.uid)
- if notify:
- if pr_action:
- pagure.lib.notify.log(
- request.project,
- topic="pull-request.%s" % pr_action,
- msg=dict(
- pullrequest=request.to_json(
- with_comments=False, public=True
- ),
- agent="pagure",
- ),
- )
- pagure.lib.query.add_pull_request_comment(
- session,
- request,
- commit=None,
- tree_id=None,
- filename=None,
- row=None,
- comment="%s" % commenttext,
- user=request.user.username,
- notify=False,
- notification=True,
- )
- session.commit()
- else:
- pagure.lib.git.update_git(request, repo=request.project)
- if with_diff:
- return (diff_commits, diff)
- else:
- return diff_commits
- def update_pull_ref(request, repo):
- """ Create or update the refs/pull/ reference in the git repo.
- """
- repopath = pagure.utils.get_repo_path(request.project)
- reponame = "%s_%s" % (request.user.user, request.uid)
- _log.info(" Adding remote: %s pointing to: %s", reponame, repopath)
- rc = RemoteCollection(repo)
- try:
- # we do rc.delete(reponame) both here and in the finally block below:
- # * here: it's useful for cases when worker was interrupted
- # on the previous execution of this function and didn't manage
- # to remove the ref
- # * in the finally clause: to remove the ref so that it doesn't stay
- # in the fork forever (as noted above, it might still stay there
- # if the worker gets interrupted, but that's not a huge deal)
- rc[reponame]
- rc.delete(reponame)
- except KeyError:
- pass
- remote = rc.create(reponame, repopath)
- try:
- _log.info(
- " Pushing refs/heads/%s to refs/pull/%s/head",
- request.branch_from,
- request.id,
- )
- refname = "+refs/heads/%s:refs/pull/%s/head" % (
- request.branch_from,
- request.id,
- )
- PagureRepo.push(remote, refname)
- finally:
- rc.delete(reponame)
- def get_git_tags(project, with_commits=False):
- """ Returns the list of tags created in the git repositorie of the
- specified project.
- """
- repopath = pagure.utils.get_repo_path(project)
- repo_obj = PagureRepo(repopath)
- if with_commits:
- tags = {}
- for tag in repo_obj.listall_references():
- if tag.startswith("refs/tags/"):
- ref = repo_obj.lookup_reference(tag)
- if ref:
- com = ref.get_object()
- if com:
- tags[tag.split("refs/tags/")[1]] = com.oid.hex
- else:
- tags = [
- tag.split("refs/tags/")[1]
- for tag in repo_obj.listall_references()
- if tag.startswith("refs/tags/")
- ]
- return tags
- def get_git_tags_objects(project):
- """ Returns the list of references of the tags created in the git
- repositorie the specified project.
- The list is sorted using the time of the commit associated to the tag """
- repopath = pagure.utils.get_repo_path(project)
- repo_obj = PagureRepo(repopath)
- tags = {}
- for tag in repo_obj.listall_references():
- if "refs/tags/" in tag and repo_obj.lookup_reference(tag):
- commit_time = None
- try:
- theobject = repo_obj[repo_obj.lookup_reference(tag).target]
- except ValueError:
- theobject = None
- objecttype = ""
- if isinstance(theobject, pygit2.Tag):
- underlying_obj = theobject.get_object()
- if isinstance(underlying_obj, pygit2.Tree):
- continue
- commit_time = underlying_obj.commit_time
- objecttype = "tag"
- elif isinstance(theobject, pygit2.Commit):
- commit_time = theobject.commit_time
- objecttype = "commit"
- tags[commit_time] = {
- "object": theobject,
- "tagname": tag.replace("refs/tags/", ""),
- "date": commit_time,
- "objecttype": objecttype,
- "head_msg": None,
- "body_msg": None,
- }
- if objecttype == "tag":
- head_msg, _, body_msg = tags[commit_time][
- "object"
- ].message.partition("\n")
- if body_msg.strip().endswith("\n-----END PGP SIGNATURE-----"):
- body_msg = body_msg.rsplit(
- "-----BEGIN PGP SIGNATURE-----", 1
- )[0].strip()
- tags[commit_time]["head_msg"] = head_msg
- tags[commit_time]["body_msg"] = body_msg
- sorted_tags = []
- for tag in sorted(tags, reverse=True):
- sorted_tags.append(tags[tag])
- return sorted_tags
- def log_commits_to_db(session, project, commits, gitdir):
- """ Log the given commits to the DB. """
- repo_obj = PagureRepo(gitdir)
- for commitid in commits:
- try:
- commit = repo_obj[commitid]
- except ValueError:
- continue
- try:
- author_obj = pagure.lib.query.get_user(
- session, commit.author.email
- )
- except pagure.exceptions.PagureException:
- author_obj = None
- date_created = arrow.get(commit.commit_time)
- log = model.PagureLog(
- user_id=author_obj.id if author_obj else None,
- user_email=commit.author.email if not author_obj else None,
- project_id=project.id,
- log_type="committed",
- ref_id=commit.oid.hex,
- date=date_created.date(),
- date_created=date_created.datetime,
- )
- session.add(log)
- def reinit_git(project, repofolder):
- """ Delete and recreate a git folder
- :args project: SQLAlchemy object of the project
- :args folder: The folder which contains the git repos
- like TICKETS_FOLDER for tickets and REQUESTS_FOLDER for
- pull requests
- """
- repo_path = os.path.join(repofolder, project.path)
- if not os.path.exists(repo_path):
- return
- # delete that repo
- shutil.rmtree(repo_path)
- # create it again
- pygit2.init_repository(
- repo_path, bare=True, mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP
- )
- def get_git_branches(project):
- """ Return a list of branches for the project
- :arg project: The Project instance to get the branches for
- """
- repo_path = pagure.utils.get_repo_path(project)
- repo_obj = pygit2.Repository(repo_path)
- return repo_obj.listall_branches()
- def new_git_branch(
- username, project, branch, from_branch=None, from_commit=None
- ):
- """ Create a new git branch on the project
- :arg project: The Project instance to get the branches for
- :arg from_branch: The branch to branch off of
- """
- with TemporaryClone(project, "main", "new_branch") as tempclone:
- repo_obj = tempclone.repo
- if not from_branch and not from_commit:
- from_branch = "master"
- branches = repo_obj.listall_branches()
- if from_branch:
- if from_branch not in branches:
- raise pagure.exceptions.PagureException(
- 'The "{0}" branch does not exist'.format(from_branch)
- )
- parent = get_branch_ref(repo_obj, from_branch).get_object()
- else:
- if from_commit not in repo_obj:
- raise pagure.exceptions.PagureException(
- 'The commit "{0}" does not exist'.format(from_commit)
- )
- parent = repo_obj[from_commit]
- if branch not in branches:
- repo_obj.create_branch(branch, parent)
- else:
- raise pagure.exceptions.PagureException(
- 'The branch "{0}" already exists'.format(branch)
- )
- tempclone.push(username, branch, branch)
- def delete_project_repos(project):
- """ Deletes the actual git repositories on disk or repoSpanner
- Args:
- project (Project): Project to delete repos for
- """
- for repotype in pagure.lib.query.get_repotypes():
- if project.is_on_repospanner:
- _, regioninfo = project.repospanner_repo_info(repotype)
- _log.debug("Deleting repotype %s", repotype)
- data = {
- "Reponame": project._repospanner_repo_name(
- repotype, project.repospanner_region
- )
- }
- resp = requests.post(
- "%s/admin/deleterepo" % regioninfo["url"],
- json=data,
- verify=regioninfo["ca"],
- cert=(
- regioninfo["admin_cert"]["cert"],
- regioninfo["admin_cert"]["key"],
- ),
- )
- resp.raise_for_status()
- resp = resp.json()
- _log.debug("Response json: %s", resp)
- if not resp["Success"]:
- raise Exception(
- "Error in repoSpanner API call: %s" % resp["Error"]
- )
- else:
- repopath = project.repopath(repotype)
- if repopath is None:
- continue
- try:
- shutil.rmtree(repopath)
- except Exception:
- _log.exception(
- "Failed to remove repotype %s for %s",
- repotype,
- project.fullname,
- )
- def set_up_project_hooks(project, region, hook=None):
- """ Makes sure the git repositories for a project have their hooks setup.
- Args:
- project (model.Project): Project to set up hooks for
- region (string or None): repoSpanner region to set hooks up for
- hook (string): The hook ID to set up in repoSpanner (tests only)
- """
- if region is None:
- # This repo is not on repoSpanner, create hooks locally
- pagure.hooks.BaseHook.set_up(project)
- else:
- regioninfo = pagure_config["REPOSPANNER_REGIONS"].get(region)
- if not regioninfo:
- raise ValueError(
- "Invalid repoSpanner region %s looked up" % region
- )
- if not hook:
- hook = regioninfo["hook"]
- if not hook:
- # No hooks to set up for this region
- return
- for repotype in pagure.lib.query.get_repotypes():
- data = {
- "Reponame": project._repospanner_repo_name(repotype, region),
- "UpdateRequest": {
- "hook-prereceive": hook,
- "hook-update": hook,
- "hook-postreceive": hook,
- },
- }
- resp = requests.post(
- "%s/admin/editrepo" % regioninfo["url"],
- json=data,
- verify=regioninfo["ca"],
- cert=(
- regioninfo["admin_cert"]["cert"],
- regioninfo["admin_cert"]["key"],
- ),
- )
- resp.raise_for_status()
- resp = resp.json()
- _log.debug("Response json: %s", resp)
- if not resp["Success"]:
- raise Exception(
- "Error in repoSpanner API call: %s" % resp["Error"]
- )
- def _create_project_repo(project, region, templ, ignore_existing, repotype):
- """ Creates a single specific git repository on disk or repoSpanner
- Args:
- project (Project): Project to create repos for
- region (string or None): repoSpanner region to create the repos in
- templ (string): Template directory, only valid for non-repoSpanner
- ignore_existing (bool): Whether a repo already existing is fatal
- repotype (string): Repotype to create
- Returns: (string or None): Directory created
- """
- if region:
- # repoSpanner creation
- regioninfo = pagure_config["REPOSPANNER_REGIONS"][region]
- # First create the repository on the repoSpanner region
- _log.debug("Creating repotype %s", repotype)
- data = {
- "Reponame": project._repospanner_repo_name(repotype, region),
- "Public": False,
- }
- resp = requests.post(
- "%s/admin/createrepo" % regioninfo["url"],
- json=data,
- verify=regioninfo["ca"],
- cert=(
- regioninfo["admin_cert"]["cert"],
- regioninfo["admin_cert"]["key"],
- ),
- )
- resp.raise_for_status()
- resp = resp.json()
- _log.debug("Response json: %s", resp)
- if not resp["Success"]:
- if "already exists" in resp["Error"]:
- if ignore_existing:
- return None
- else:
- raise pagure.exceptions.RepoExistsException(resp["Error"])
- raise Exception(
- "Error in repoSpanner API call: %s" % resp["Error"]
- )
- return None
- else:
- # local repo
- repodir = project.repopath(repotype)
- if repodir is None:
- # This repo type is disabled
- return None
- if os.path.exists(repodir):
- if not ignore_existing:
- raise pagure.exceptions.RepoExistsException(
- "The %s repo %s already exists" % (repotype, project.path)
- )
- else:
- return None
- if repotype == "main":
- pygit2.init_repository(repodir, bare=True, template_path=templ)
- if not project.private:
- # Make the repo exportable via apache
- http_clone_file = os.path.join(repodir, "git-daemon-export-ok")
- if not os.path.exists(http_clone_file):
- with open(http_clone_file, "w"):
- pass
- else:
- pygit2.init_repository(
- repodir,
- bare=True,
- mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP,
- )
- return repodir
- def create_project_repos(project, region, templ, ignore_existing):
- """ Creates the actual git repositories on disk or repoSpanner
- Args:
- project (Project): Project to create repos for
- region (string or None): repoSpanner region to create the repos in
- templ (string): Template directory, only valid for non-repoSpanner
- ignore_existing (bool): Whether a repo already existing is fatal
- """
- if region and templ:
- raise Exception("repoSpanner is incompatible with template directory")
- created_dirs = []
- try:
- for repotype in pagure.lib.query.get_repotypes():
- created = _create_project_repo(
- project, region, templ, ignore_existing, repotype
- )
- if created:
- created_dirs.append(created)
- except Exception:
- for created in created_dirs:
- shutil.rmtree(created)
- raise
- set_up_project_hooks(project, region)
- def get_stats_patch(patch):
- """ Returns some statistics about a given patch.
- These stats include:
- status: if the file was added (A), deleted (D), modified (M) or
- renamed (R)
- old_path: the path to the old file
- new_path: the path to the new file
- lines_added: the number of lines added in this patch
- lines_removed: the number of lines removed in this patch
- All these information are returned in a dict.
- Args:
- patch (pygit2.Patch): the patch object to get stats on
- Returns: a dict with the stats described above
- Raises (pagure.exceptions.PagureException): if for some reason (likely
- a change in pygit2's API) this function does not manage to gather
- all the stats it should
- """
- output = {
- "lines_added": patch.line_stats[1],
- "lines_removed": patch.line_stats[2],
- "new_path": None,
- "old_path": None,
- "status": None,
- "new_id": None,
- "old_id": None,
- }
- if hasattr(patch, "new_file_path"):
- # Older pygit2
- status = patch.status
- if patch.new_file_path != patch.old_file_path:
- status = "R"
- output["status"] = status
- output["new_path"] = patch.new_file_path
- output["old_path"] = patch.old_file_path
- output["new_id"] = str(patch.new_id)
- output["old_id"] = str(patch.old_id)
- elif hasattr(patch, "delta"):
- # Newer pygit2
- if patch.delta.new_file.mode == 0 and patch.delta.old_file.mode in [
- 33188,
- 33261,
- ]:
- status = "D"
- elif (
- patch.delta.new_file.mode in [33188, 33261]
- and patch.delta.old_file.mode == 0
- ):
- status = "A"
- elif patch.delta.new_file.mode in [
- 33188,
- 33261,
- ] and patch.delta.old_file.mode in [33188, 33261]:
- status = "M"
- if patch.delta.new_file.path != patch.delta.old_file.path:
- status = "R"
- output["status"] = status
- output["new_path"] = patch.delta.new_file.path
- output["new_id"] = str(patch.delta.new_file.id)
- output["old_path"] = patch.delta.old_file.path
- output["old_id"] = str(patch.delta.old_file.id)
- if None in output.values(): # pragma: no-cover
- raise pagure.exceptions.PagureException(
- "Unable to properly retrieve the stats for this patch"
- )
- return output
- def generate_archive(project, commit, tag, name, archive_fmt):
- """ Generate the desired archive of the specified project for the
- specified commit with the given name and archive format.
- Args:
- project (pagure.lib.model.Project): the project's repository from
- which to generate the archive
- commit (str): the commit hash to generate the archive of
- name (str): the name to give to the archive
- archive_fmt (str): the format of the archive to generate, can be
- either gzip or tag or tar.gz
- Returns: None
- Raises (pagure.exceptions.PagureException): if an un-supported archive
- format is specified
- """
- def _exclude_git(filename):
- return ".git" in filename
- with TemporaryClone(project, "main", "archive", parent=name) as tempclone:
- repo_obj = tempclone.repo
- commit_obj = repo_obj[commit]
- repo_obj.checkout_tree(commit_obj.tree)
- archive_folder = pagure_config.get("ARCHIVE_FOLDER")
- tag_path = ""
- if tag:
- tag_path = os.path.join("tags", tag)
- target_path = os.path.join(
- archive_folder, project.fullname, tag_path, commit
- )
- if not os.path.exists(target_path):
- os.makedirs(target_path)
- fullpath = os.path.join(target_path, name)
- if archive_fmt == "tar":
- with tarfile.open(name=fullpath + ".tar", mode="w") as tar:
- tar.add(
- name=tempclone.repopath, exclude=_exclude_git, arcname=name
- )
- elif archive_fmt == "tar.gz":
- with tarfile.open(name=fullpath + ".tar.gz", mode="w:gz") as tar:
- tar.add(
- name=tempclone.repopath, exclude=_exclude_git, arcname=name
- )
- elif archive_fmt == "zip":
- # Code from /usr/lib64/python2.7/zipfile.py adjusted for our
- # needs
- def addToZip(zf, path, zippath):
- if _exclude_git(path):
- return
- if os.path.isfile(path):
- zf.write(path, zippath, zipfile.ZIP_DEFLATED)
- elif os.path.isdir(path):
- if zippath:
- zf.write(path, zippath)
- for nm in os.listdir(path):
- if _exclude_git(path):
- continue
- addToZip(
- zf,
- os.path.join(path, nm),
- os.path.join(zippath, nm),
- )
- with zipfile.ZipFile(fullpath + ".zip", "w") as zipstream:
- addToZip(zipstream, tempclone.repopath, name)
- else:
- raise pagure.exceptions.PagureException(
- "Un-support archive format requested: %s", archive_fmt
- )
- def mirror_pull_project(session, project, debug=False):
- """ Mirror locally a project from a remote URL. """
- remote = project.mirrored_from
- repopath = tempfile.mkdtemp(prefix="pagure-mirror_in-")
- lclrepopath = pagure.utils.get_repo_path(project)
- def _run_command(command, logs):
- _log.info("Running the command: %s" % command)
- if debug:
- print("Running the command: %s" % command)
- print(" Running in: %s" % repopath)
- (stdout, stderr) = pagure.lib.git.read_git_lines(
- command, abspath=repopath, error=True
- )
- log = "Output from %s:\n stdout: %s\n stderr: %s" % (
- command,
- stdout,
- stderr,
- )
- logs.append(log)
- if debug:
- print(log)
- return logs
- try:
- # Pull
- logs = []
- logs = _run_command(["clone", "--mirror", remote, "."], logs)
- logs = _run_command(["remote", "add", "local", lclrepopath], logs)
- # Push the changes
- _log.info("Pushing")
- if debug:
- print("Pushing to the local git repo")
- extra = {}
- if project.is_on_repospanner:
- regioninfo = pagure_config["REPOSPANNER_REGIONS"][
- project.repospanner_region
- ]
- extra.update(
- {
- "username": "pagure",
- "repotype": "main",
- "project_name": project.name,
- "project_user": project.user.username
- if project.is_fork
- else "",
- "project_namespace": project.namespace or "",
- }
- )
- args = []
- for opt in extra:
- args.extend(["--extra", opt, extra[opt]])
- command = [
- "git",
- "-c",
- "protocol.ext.allow=always",
- "push",
- "ext::%s %s %s"
- % (
- pagure_config["REPOBRIDGE_BINARY"],
- " ".join(args),
- project._repospanner_repo_name("main"),
- ),
- "--repo",
- repopath,
- ]
- environ = {
- "USER": "pagure",
- "REPOBRIDGE_CONFIG": ":environment:",
- "REPOBRIDGE_BASEURL": regioninfo["url"],
- "REPOBRIDGE_CA": regioninfo["ca"],
- "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"],
- "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"],
- }
- else:
- command = ["git", "push", "local", "--mirror"]
- environ = {}
- _log.debug("Running a git push to %s", project.fullname)
- env = os.environ.copy()
- env["GL_USER"] = "pagure"
- env["GL_BYPASS_ACCESS_CHECKS"] = "1"
- if pagure_config.get("GITOLITE_HOME"):
- env["HOME"] = pagure_config["GITOLITE_HOME"]
- env.update(environ)
- env.update(extra)
- out = subprocess.check_output(
- command, cwd=repopath, stderr=subprocess.STDOUT, env=env
- )
- log = "Output from %s:" % command
- logs.append(log)
- logs.append(out)
- _log.debug("Output: %s" % out)
- project.mirrored_from_last_log = "\n".join(logs)
- session.add(project)
- session.commit()
- _log.info("\n".join(logs))
- except subprocess.CalledProcessError as err:
- _log.debug(
- "Rebase FAILED: {cmd} returned code {code} with the "
- "following output: {output}".format(
- cmd=err.cmd, code=err.returncode, output=err.output
- )
- )
- # This should never really happen, since we control the repos, but
- # this way, we can be sure to get the output logged
- remotes = []
- for line in err.output.decode("utf-8").split("\n"):
- _log.info("Remote line: %s", line)
- if line.startswith("remote: "):
- _log.debug("Remote: %s" % line)
- remotes.append(line[len("remote: ") :].strip())
- if remotes:
- _log.info("Remote rejected with: %s" % remotes)
- raise pagure.exceptions.PagurePushDenied(
- "Remote hook declined the push: %s" % "\n".join(remotes)
- )
- else:
- # Something else happened, pass the original
- _log.exception("Error pushing. Output: %s", err.output)
- raise
- finally:
- shutil.rmtree(repopath)
|