Add an API endpoint to view the content of a git repo

This will return a JSON blob with the content of a git repo or one of
its sub-folder or file.
It will include a link where one can find the actual file or content of
the folder.

Fixes https://pagure.io/pagure/issue/4808

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

pagure/api/__init__.py

@@ -127,6 +127,9 @@ class APIERROR(enum.Enum):
     EINVALIDPERPAGEVALUE = "The per_page value must be between 1 and 100"
     EGITERROR = "An error occurred during a git operation"
     ENOCOMMIT = "No such commit found in this repository"
+    EEMPTYGIT = "This git repository is empty"
+    EBRANCHNOTFOUND = "Branch not found in this git repository"
+    EFILENOTFOUND = "File not found in this git repository"
     ENOTHIGHENOUGH = (
         "You do not have sufficient permissions to perform this action"
     )

pagure/api/project.py

@@ -686,6 +686,220 @@ def api_git_branches(repo, username=None, namespace=None):
     )
 
 
+@API.route("/<repo>/tree")
+@API.route("/<repo>/tree/<path:identifier>")
+@API.route("/<repo>/tree/<path:identifier>/f/<path:filename>")
+@API.route("/<namespace>/<repo>/tree")
+@API.route("/<namespace>/<repo>/tree/<path:identifier>")
+@API.route("/<namespace>/<repo>/tree/<path:identifier>/f/<path:filename>")
+@API.route("/fork/<username>/<repo>/tree")
+@API.route("/fork/<username>/<repo>/tree/<path:identifier>")
+@API.route("/fork/<username>/<repo>/tree/<path:identifier>/f/<path:filename>")
+@API.route("/fork/<username>/<namespace>/<repo>/tree")
+@API.route("/fork/<username>/<namespace>/<repo>/tree/<path:identifier>/")
+@API.route(
+    "/fork/<username>/<namespace>/<repo>/tree/<path:identifier>/"
+    "f/<path:filename>"
+)
+@api_method
+def api_view_file(
+    repo, username=None, namespace=None, identifier=None, filename=None
+):
+    """
+    List files in a project
+    -----------------------
+    Lists the files present in a project or one of its subfolder.
+
+    ::
+
+        GET /api/0/<repo>tree
+        GET /api/0/<repo>tree/master
+        GET /api/0/<repo>tree/master/f/<filename>
+        GET /api/0/<repo>tree/master/f/<folder>/
+        GET /api/0/<repo>tree/master/f/<folder1>/<folder2>/<filename>
+
+
+    ::
+
+        GET /api/0/fork/<username>/<repo>tree
+        GET /api/0/fork/<username>/<repo>tree/master
+        GET /api/0/fork/<username>/<repo>tree/master/f/<filename>
+        GET /api/0/fork/<username>/<repo>tree/master/f/<folder>/
+        GET /api/0/fork/<username>/<repo>tree/master/f/<folder1>/<folder2>/<filename>
+
+
+    Sample response
+    ^^^^^^^^^^^^^^^
+
+    ::
+
+        {
+          "content": [
+            {
+              "content_url": "https://pagure.io/api/0/pagure/tree/master/f/alembic",
+              "name": "alembic",
+              "path": "alembic",
+              "type": "folder"
+            },
+            {
+              "content_url": "https://pagure.io/api/0/pagure/tree/master/f/fedmsg.d",
+              "name": "fedmsg.d",
+              "path": "fedmsg.d",
+              "type": "folder"
+            },
+            {
+              "content_url": "https://pagure.io/pagure/raw/master/f/tox.ini",
+              "name": "tox.ini",
+              "path": "tox.ini",
+              "type": "file"
+            }
+          ],
+          "name": null,
+          "type": "folder"
+        }
+
+        {
+          "content": [
+            {
+              "content_url": "https://pagure.io/pagure/raw/master/f/fedmsg.d/pagure.py",
+              "name": "pagure.py",
+              "path": "fedmsg.d/pagure.py",
+              "type": "file"
+            },
+            {
+              "content_url": "https://pagure.io/pagure/raw/master/f/fedmsg.d/pagure_ci.py",
+              "name": "pagure_ci.py",
+              "path": "fedmsg.d/pagure_ci.py",
+              "type": "file"
+            }
+          ],
+          "name": "fedmsg.d",
+          "type": "folder"
+        }
+
+    """  # noqa
+    repo = _get_repo(repo, username, namespace)
+    repopath = pagure.utils.get_repo_path(repo)
+    repo_obj = pygit2.Repository(repopath)
+
+    if repo_obj.is_empty:
+        raise pagure.exceptions.APIError(404, error_code=APIERROR.EEMPTYGIT)
+
+    if identifier in repo_obj.listall_branches():
+        branchname = identifier
+        branch = repo_obj.lookup_branch(identifier)
+        commit = branch.peel(pygit2.Commit)
+    else:
+        try:
+            commit = repo_obj.get(identifier)
+            branchname = identifier
+        except (ValueError, TypeError):
+            # If an identifier was provided, bail, the provided info is wrong
+            if identifier:
+                raise pagure.exceptions.APIError(
+                    404, error_code=APIERROR.EFILENOTFOUND
+                )
+            # If it's not a commit id then it's part of the filename
+            if not repo_obj.head_is_unborn:
+                branchname = repo_obj.head.shorthand
+                commit = repo_obj[repo_obj.head.target]
+
+    if isinstance(commit, pygit2.Tag):
+        commit = commit.peel(pygit2.Commit)
+
+    tree = None
+    if isinstance(commit, pygit2.Tree):
+        tree = commit
+    elif isinstance(commit, pygit2.Commit):
+        tree = commit.tree
+
+    if tree and not filename:
+        content = sorted(tree, key=lambda x: x.filemode)
+    elif tree and commit and not isinstance(commit, pygit2.Blob):
+        content = pagure.utils.__get_file_in_tree(
+            repo_obj, tree, filename.split("/"), bail_on_tree=True
+        )
+        if not content:
+            raise pagure.exceptions.APIError(
+                404, error_code=APIERROR.EFILENOTFOUND
+            )
+        content = repo_obj[content.oid]
+    else:
+        content = commit
+
+    if not content:
+        raise pagure.exceptions.APIError(
+            404, error_code=APIERROR.EFILENOTFOUND
+        )
+
+    output_type = "tree"
+    if isinstance(content, pygit2.Blob):
+        output_type = "file"
+    elif isinstance(content, pygit2.Commit):
+        raise pagure.exceptions.APIError(
+            404, error_code=APIERROR.EFILENOTFOUND
+        )
+
+    if output_type == "file":
+        output = {
+            "type": "file",
+            "name": filename,
+            "content_url": flask.url_for(
+                "ui_ns.view_raw_file",
+                repo=repo.name,
+                username=username,
+                namespace=repo.namespace,
+                identifier=branchname,
+                filename=filename,
+                _external=True,
+            ),
+        }
+    else:
+        content_list = []
+        for entry in content:
+            path = filename + "/" + entry.name if filename else entry.name
+            url_content = flask.url_for(
+                "api_ns.api_view_file",
+                repo=repo.name,
+                username=username,
+                namespace=repo.namespace,
+                identifier=branchname,
+                filename=path,
+                _external=True,
+            )
+            if entry.filemode == 16384:
+                file_type = "folder"
+            elif entry.filemode == 40960:
+                file_type = "link"
+            elif entry.filemode == 57344:
+                file_type = "submodule"
+            else:
+                file_type = "file"
+                url_content = flask.url_for(
+                    "ui_ns.view_raw_file",
+                    repo=repo.name,
+                    username=username,
+                    namespace=repo.namespace,
+                    identifier=branchname,
+                    filename=path,
+                    _external=True,
+                )
+            tmp = {
+                "type": file_type,
+                "name": entry.name,
+                "path": path,
+                "content_url": url_content,
+            }
+            content_list.append(tmp)
+        output = {
+            "type": "folder",
+            "name": filename,
+            "content": content_list,
+        }
+
+    return flask.jsonify(output)
+
+
 @API.route("/projects")
 @api_method
 def api_projects():

tests/test_pagure_flask_api.py

@@ -176,13 +176,16 @@ class PagureFlaskApitests(tests.SimplePagureTest):
         output = self.app.get("/api/0/-/error_codes")
         self.assertEqual(output.status_code, 200)
         data = json.loads(output.get_data(as_text=True))
-        self.assertEqual(len(data), 42)
+        self.assertEqual(len(data), 45)
         self.assertEqual(
             sorted(data.keys()),
             sorted(
                 [
+                    "EBRANCHNOTFOUND",
                     "EDATETIME",
                     "EDBERROR",
+                    "EEMPTYGIT",
+                    "EFILENOTFOUND",
                     "EGITERROR",
                     "EINVALIDISSUEFIELD",
                     "EINVALIDISSUEFIELD_LINK",

tests/test_pagure_flask_api_project_view_file.py

@@ -0,0 +1,328 @@
+# -*- coding: utf-8 -*-
+
+"""
+ (c) 2020 - Copyright Red Hat Inc
+
+ Authors:
+   Pierre-Yves Chibon <pingou@pingoured.fr>
+
+"""
+
+from __future__ import unicode_literals, absolute_import
+
+import datetime
+import json
+import unittest
+import shutil
+import sys
+import tempfile
+import os
+
+import pygit2
+from celery.result import EagerResult
+from mock import patch, Mock
+
+sys.path.insert(
+    0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
+)
+
+import pagure.flask_app
+import pagure.lib.query
+import tests
+from pagure.lib.repo import PagureRepo
+
+
+class PagureFlaskApiProjectViewFiletests(tests.Modeltests):
+    """ Tests for the flask API of pagure for issue """
+
+    maxDiff = None
+
+    def setUp(self):
+        super(PagureFlaskApiProjectViewFiletests, self).setUp()
+        tests.create_projects(self.session)
+        tests.create_projects_git(os.path.join(self.path, "repos"), bare=True)
+        tests.add_readme_git_repo(os.path.join(self.path, "repos", "test.git"))
+
+    def test_view_file_invalid_project(self):
+        output = self.app.get("/api/0/invalid/tree")
+        self.assertEqual(output.status_code, 404)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data, {"error": "Project not found", "error_code": "ENOPROJECT"}
+        )
+
+    def test_view_file_invalid_ref_and_path(self):
+        output = self.app.get("/api/0/test/tree/branchname/f/foldername")
+        self.assertEqual(output.status_code, 404)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "error": "File not found in this git repository",
+                "error_code": "EFILENOTFOUND",
+            },
+        )
+
+    def test_view_file_empty_project(self):
+        output = self.app.get("/api/0/test2/tree")
+        self.assertEqual(output.status_code, 404)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "error": "This git repository is empty",
+                "error_code": "EEMPTYGIT",
+            },
+        )
+
+    def test_view_file_basic(self):
+        output = self.app.get("/api/0/test/tree")
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/test/raw/master/"
+                        "f/README.rst",
+                        "name": "README.rst",
+                        "path": "README.rst",
+                        "type": "file",
+                    }
+                ],
+                "name": None,
+                "type": "folder",
+            },
+        )
+
+    def test_view_file_with_folder(self):
+        tests.add_content_git_repo(
+            os.path.join(self.path, "repos", "test.git")
+        )
+        output = self.app.get("/api/0/test/tree")
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/api/0/test/tree/"
+                        "master/f/folder1",
+                        "name": "folder1",
+                        "path": "folder1",
+                        "type": "folder",
+                    },
+                    {
+                        "content_url": "http://localhost/test/raw/master/f/"
+                        "README.rst",
+                        "name": "README.rst",
+                        "path": "README.rst",
+                        "type": "file",
+                    },
+                    {
+                        "content_url": "http://localhost/test/raw/master/f/"
+                        "sources",
+                        "name": "sources",
+                        "path": "sources",
+                        "type": "file",
+                    },
+                ],
+                "name": None,
+                "type": "folder",
+            },
+        )
+
+    def test_view_file_specific_file(self):
+        tests.add_content_git_repo(
+            os.path.join(self.path, "repos", "test.git")
+        )
+        output = self.app.get("/api/0/test/tree/master/f/README.rst")
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content_url": "http://localhost/test/raw/master/f/README.rst",
+                "name": "README.rst",
+                "type": "file",
+            },
+        )
+
+    def test_view_file_invalid_ref(self):
+        tests.add_content_git_repo(
+            os.path.join(self.path, "repos", "test.git")
+        )
+        output = self.app.get("/api/0/test/tree/invalid/f/folder1")
+        print(output.data)
+        self.assertEqual(output.status_code, 404)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "error": "File not found in this git repository",
+                "error_code": "EFILENOTFOUND",
+            },
+        )
+
+    def test_view_file_invalid_folder(self):
+        tests.add_content_git_repo(
+            os.path.join(self.path, "repos", "test.git")
+        )
+        output = self.app.get("/api/0/test/tree/master/f/inv/invalid")
+        self.assertEqual(output.status_code, 404)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "error": "File not found in this git repository",
+                "error_code": "EFILENOTFOUND",
+            },
+        )
+
+    def test_view_file_valid_branch(self):
+        tests.add_content_git_repo(
+            os.path.join(self.path, "repos", "test.git")
+        )
+        output = self.app.get("/api/0/test/tree/master/f/folder1")
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/api/0/test/tree/"
+                        "master/f/folder1/folder2",
+                        "name": "folder2",
+                        "path": "folder1/folder2",
+                        "type": "folder",
+                    }
+                ],
+                "name": "folder1",
+                "type": "folder",
+            },
+        )
+
+    def test_view_file_non_ascii_name(self):
+        # View file with a non-ascii name
+        tests.add_commit_git_repo(
+            os.path.join(self.path, "repos", "test.git"),
+            ncommits=1,
+            filename="Šource",
+        )
+        output = self.app.get("/api/0/test/tree")
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True).encode("utf-8"))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/test/raw/master/f/"
+                        "README.rst",
+                        "name": "README.rst",
+                        "path": "README.rst",
+                        "type": "file",
+                    },
+                    {
+                        "content_url": "http://localhost/test/raw/master/f/%C5%A0ource",
+                        "name": "Šource",
+                        "path": "Šource",
+                        "type": "file",
+                    },
+                ],
+                "name": None,
+                "type": "folder",
+            },
+        )
+
+    def test_view_file_from_commit(self):
+        repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git"))
+        commit = repo.revparse_single("HEAD")
+
+        output = self.app.get("/api/0/test/tree/%s" % commit.oid.hex)
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/test/raw/"
+                        "%s/f/README.rst" % commit.oid.hex,
+                        "name": "README.rst",
+                        "path": "README.rst",
+                        "type": "file",
+                    }
+                ],
+                "name": None,
+                "type": "folder",
+            },
+        )
+
+    def test_view_file_from_tree(self):
+        tests.add_content_git_repo(
+            os.path.join(self.path, "repos", "test.git")
+        )
+        repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git"))
+        commit = repo.revparse_single("HEAD")
+
+        output = self.app.get(
+            "/api/0/test/tree/%s/f/folder1" % commit.tree.oid.hex
+        )
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/api/0/test/tree/"
+                        "%s/f/folder1/folder2" % commit.tree.oid.hex,
+                        "name": "folder2",
+                        "path": "folder1/folder2",
+                        "type": "folder",
+                    }
+                ],
+                "name": "folder1",
+                "type": "folder",
+            },
+        )
+
+    def test_view_file_from_tag_hex(self):
+        repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git"))
+        commit = repo.revparse_single("HEAD")
+        tagger = pygit2.Signature("Alice Doe", "adoe@example.com", 12347, 0)
+        tag = repo.create_tag(
+            "v1.0_tag",
+            commit.oid.hex,
+            pygit2.GIT_OBJ_COMMIT,
+            tagger,
+            "Release v1.0",
+        )
+
+        output = self.app.get("/api/0/test/tree/%s" % tag.hex)
+        self.assertEqual(output.status_code, 200)
+        data = json.loads(output.get_data(as_text=True))
+        self.assertDictEqual(
+            data,
+            {
+                "content": [
+                    {
+                        "content_url": "http://localhost/test/raw/"
+                        "%s/f/README.rst" % tag.hex,
+                        "name": "README.rst",
+                        "path": "README.rst",
+                        "type": "file",
+                    }
+                ],
+                "name": None,
+                "type": "folder",
+            },
+        )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)