pagure/pagure/pfmarkdown.py

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
# USA.

""" Pagure-flavored Markdown

Author: Ralph Bean <rbean@redhat.com>
        Pierre-Yves Chibon <pingou@pingoured.fr>
"""

from __future__ import absolute_import, unicode_literals

import logging
import re

import flask
import markdown.inlinepatterns
import markdown.postprocessors
import markdown.preprocessors
import markdown.util
import pygit2
import six
import xml.etree.ElementTree as etree

import pagure.lib.query
from pagure.config import config as pagure_config

try:
    from markdown.inlinepatterns import ImagePattern as ImagePattern

    MK_VERSION = 2
except ImportError:
    from markdown.inlinepatterns import ImageInlineProcessor as ImagePattern

    MK_VERSION = 3

_log = logging.getLogger(__name__)


# the (?<!\w) (and variants) we use a lot in all these regexes is a
# negative lookbehind assertion. It means 'match when the preceding
# character is not in the \w class'. This stops us from starting a
# match in the middle of a word (e.g. someone@something in the
# MENTION_RE regex). Note that it is a zero-length match - it does
# not capture or consume any of the string - and it does not appear
# as a group for the match object.
MENTION_RE = r"(?<![\w\-\"\'\`\$\!\*\+#%&/=^{}|~])@(\w+)"
# Each line below correspond to a line of the regex:
#  1) Don't start matching in the middle of a word
#  2) See if there is a `forks/` at the start
#  3) See if we have a `user/`
#  4) See if we have a `namespace/`
#  5) Get the last part `project`
#  6) Get the identifier `#<id>`
EXPLICIT_LINK_RE = (
    r"(?<!\w)"
    r"(fork[s]?/)?"
    r"([a-zA-Z0-9_-]*?/)?"
    r"([a-zA-Z0-9_-]*?/)?"
    r"([a-zA-Z0-9_-]+)"
    r"#(?P<id>[0-9]+)"
)
COMMIT_LINK_RE = (
    r"(?<!\w)"
    r"(fork[s]?/)?"
    r"([a-zA-Z0-9_-]*?/)?"
    r"([a-zA-Z0-9_-]*?/)?"
    r"([a-zA-Z0-9_-]+)"
    r"#(?P<id>[\w]{40})"
)
# PREPROCIMPLLINK is used by ImplicitIssuePreprocessor to replace the
# '#' when a line starts with an implicit issue link, to prevent
# markdown parsing it as a header; we have to handle it here
IMPLICIT_ISSUE_RE = r"(?<!\w)(?:PREPROCIMPLLINK|#)([0-9]+)"
IMPLICIT_PR_RE = r"(?<!\w)PR#([0-9]+)"
IMPLICIT_COMMIT_RE = r"(?<![<\w#])([a-f0-9]{7,40})"
STRIKE_THROUGH_RE = r"~~(.*?)~~"


class MentionPattern(markdown.inlinepatterns.Pattern):
    """@user pattern class."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("MentionPattern: %s", m.groups())

        name = markdown.util.AtomicString(m.group(2))
        text = "@%s" % name
        user = pagure.lib.query.search_user(flask.g.session, username=name)
        if not user:
            return text

        element = etree.Element("a")
        base_url = pagure_config["APP_URL"]
        if base_url.endswith("/"):
            base_url = base_url[:-1]
        url = "%s/user/%s" % (base_url, user.username)
        element.set("href", url)
        element.text = text
        return element


class ExplicitLinkPattern(markdown.inlinepatterns.Pattern):
    """Explicit link pattern."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("ExplicitLinkPattern: %s", m.groups())

        is_fork = m.group(2)
        user = m.group(3)
        namespace = m.group(4)
        repo = m.group(5)
        idx = m.group(6)
        text = "%s#%s" % (repo, idx)

        if not is_fork and user:
            namespace = user
            user = None

        if namespace:
            namespace = namespace.rstrip("/")
            text = "%s/%s" % (namespace, text)
        if user:
            user = user.rstrip("/")
            text = "%s/%s" % (user.rstrip("/"), text)

        try:
            idx = int(idx)
        except (ValueError, TypeError):
            return text

        issue = _issue_exists(user, namespace, repo, idx)
        if issue:
            return _obj_anchor_tag(user, namespace, repo, issue, text)

        request = _pr_exists(user, namespace, repo, idx)
        if request:
            return _obj_anchor_tag(user, namespace, repo, request, text)

        return text


class CommitLinkPattern(markdown.inlinepatterns.Pattern):
    """Commit link pattern."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("CommitLinkPattern: %s", m.groups())

        is_fork = m.group(2)
        user = m.group(3)
        namespace = m.group(4)
        repo = m.group(5)
        commitid = m.group(6)
        text = "%s#%s" % (repo, commitid)

        if not is_fork and user:
            namespace = user
            user = None

        if namespace:
            namespace = namespace.rstrip("/")
            text = "%s/%s" % (namespace, text)
        if user:
            user = user.rstrip("/")
            text = "%s/%s" % (user.rstrip("/"), text)

        if pagure.lib.query.search_projects(
            flask.g.session,
            username=user,
            fork=is_fork,
            namespace=namespace,
            pattern=repo,
        ):
            return _obj_anchor_tag(user, namespace, repo, commitid, text)

        return text


class ImplicitIssuePreprocessor(markdown.preprocessors.Preprocessor):
    """
    Preprocessor which handles lines starting with an implicit
    link. We have to modify these so that markdown doesn't interpret
    them as headers.
    """

    def run(self, lines):
        """
        If a line starts with an implicit issue link like #152,
        we replace the # with PREPROCIMPLLINK. This prevents markdown
        parsing the line as a header. ImplicitIssuePattern will catch
        and parse the text later. Otherwise, we change nothing.
        """
        _log.debug("ImplicitIssuePreprocessor")
        # match a # character, then any number of digits
        regex = re.compile(r"#([0-9]+)")
        new_lines = []
        for line in lines:
            # avoid calling the regex if line doesn't start with #
            if line.startswith("#"):
                match = regex.match(line)
                if match:
                    idx = int(match.group(1))
                    # we have to check if this is a real issue or PR now.
                    # we can't just 'tag' the text somehow and leave it to
                    # the pattern to check, as if it's *not* one we want
                    # the line treated as a header, so we need the block
                    # processor to see it unmodified.
                    try:
                        namespace, repo, user = _get_ns_repo_user()
                    except RuntimeError:
                        # non-match path, keep original line
                        new_lines.append(line)
                        continue
                    if _issue_exists(user, namespace, repo, idx) or _pr_exists(
                        user, namespace, repo, idx
                    ):
                        # tweak the text
                        new_lines.append("PREPROCIMPLLINK" + line[1:])
                        continue
            # this is a non-match path, keep original line
            new_lines.append(line)
            continue
        return new_lines


class ImplicitIssuePattern(markdown.inlinepatterns.Pattern):
    """Implicit issue pattern."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("ImplicitIssuePattern: %s", m.groups())
        idx = markdown.util.AtomicString(m.group(2))
        text = "#%s" % idx
        try:
            idx = int(idx)
        except (ValueError, TypeError):
            _log.debug("Invalid integer for %s, bailing", idx)
            return text

        try:
            namespace, repo, user = _get_ns_repo_user()
        except RuntimeError:
            _log.debug("No repo found associated with this context, bailing")
            return text

        _log.debug(
            "Checking ns: %s, name: %s, user:%s for id: %s",
            namespace,
            repo,
            user,
            idx,
        )

        issue = _issue_exists(user, namespace, repo, idx)
        if issue:
            _log.debug("Linking to an issue")
            return _obj_anchor_tag(user, namespace, repo, issue, text)

        request = _pr_exists(user, namespace, repo, idx)
        if request:
            _log.debug("Linking to an PR")
            return _obj_anchor_tag(user, namespace, repo, request, text)

        _log.debug("Bailing, return text as is")
        return text


class ImplicitPRPattern(markdown.inlinepatterns.Pattern):
    """Implicit pull-request pattern."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("ImplicitPRPattern: %s", m.groups())
        idx = markdown.util.AtomicString(m.group(2))
        text = "PR#%s" % idx
        try:
            idx = int(idx)
        except (ValueError, TypeError):
            return text

        try:
            namespace, repo, user = _get_ns_repo_user()
        except RuntimeError:
            return text

        issue = _issue_exists(user, namespace, repo, idx)
        if issue:
            return _obj_anchor_tag(user, namespace, repo, issue, text)

        request = _pr_exists(user, namespace, repo, idx)
        if request:
            return _obj_anchor_tag(user, namespace, repo, request, text)

        return text


class ImplicitCommitPattern(markdown.inlinepatterns.Pattern):
    """Implicit commit pattern."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("ImplicitCommitPattern: %s", m.groups())

        githash = markdown.util.AtomicString(m.group(2))
        text = "%s" % githash

        try:
            namespace, repo, user = _get_ns_repo_user()
        except RuntimeError:
            return text

        if pagure.lib.query.search_projects(
            flask.g.session, username=user, namespace=namespace, pattern=repo
        ) and _commit_exists(user, namespace, repo, githash):
            return _obj_anchor_tag(user, namespace, repo, githash, text[:7])

        return text


class StrikeThroughPattern(markdown.inlinepatterns.Pattern):
    """~~striked~~ pattern class."""

    def handleMatch(self, m):
        """When the pattern matches, update the text."""
        _log.debug("StrikeThroughPattern: %s", m.groups())

        text = markdown.util.AtomicString(m.group(2))

        element = etree.Element("del")
        element.text = text
        return element


class AutolinkPattern2(markdown.inlinepatterns.Pattern):
    """Return a link Element given an autolink (`<http://example/com>`)."""

    def handleMatch(self, m):
        """When the pattern matches, update the text.

        :arg m: the matched object

        """
        _log.debug("AutolinkPattern2: %s", m.groups())

        url = m.group(2)
        if url.startswith("<"):
            url = url[1:]
        if url.endswith(">"):
            url = url[:-1]
        el = etree.Element("a")
        el.set("href", self.unescape(url))
        el.text = markdown.util.AtomicString(url)
        return el


class ImagePatternLazyLoad(ImagePattern):
    """Customize the image element matched for lazyloading."""

    def handleMatch(self, m, *args):
        out = super(ImagePatternLazyLoad, self).handleMatch(m, *args)
        if MK_VERSION == 3:
            el = out[0]
        else:
            el = out

        # Add a noscript tag with the untouched img tag
        noscript = etree.Element("noscript")
        noscript.append(el)

        # Modify the origina img tag
        img = etree.Element("img")
        img.set("data-src", el.get("src"))
        img.set("src", "")
        img.set("alt", el.get("alt"))
        img.set("class", "lazyload")

        # Create a global span in which we add both the new img tag and the
        # noscript one
        outel = etree.Element("span")
        outel.append(img)
        outel.append(noscript)

        output = outel
        if MK_VERSION == 3:
            output = (outel, out[1], out[2])

        return output


class EncapsulateMarkdownPostprocessor(markdown.postprocessors.Postprocessor):
    def run(self, text):
        return '<div class="markdown">' + text + "</div>"


class PagureExtension(markdown.extensions.Extension):
    def extendMarkdown(self, md, *args):
        # First, make it so that bare links get automatically linkified.
        AUTOLINK_RE = "(%s)" % "|".join(
            [
                r"<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^>]*)>",
                r"\b(?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^)<>\s]+[^.,)<>\s]",
                r"<(Ii][Rr][Cc][Ss]?://[^>]*)>",
                r"\b[Ii][Rr][Cc][Ss]?://[^)<>\s]+[^.,)<>\s]",
            ]
        )

        def _old_mardkown_way():
            markdown.inlinepatterns.AUTOLINK_RE = AUTOLINK_RE
            md.preprocessors["implicit_issue"] = ImplicitIssuePreprocessor()
            md.inlinePatterns["mention"] = MentionPattern(MENTION_RE)
            # Customize the image linking to support lazy loading
            md.inlinePatterns["image_link"] = ImagePatternLazyLoad(
                markdown.inlinepatterns.IMAGE_LINK_RE, md
            )
            md.inlinePatterns["implicit_commit"] = ImplicitCommitPattern(
                IMPLICIT_COMMIT_RE
            )
            md.inlinePatterns["commit_links"] = CommitLinkPattern(
                COMMIT_LINK_RE
            )
            md.inlinePatterns["autolink"] = AutolinkPattern2(AUTOLINK_RE, md)
            if pagure_config.get("ENABLE_TICKETS", True):
                md.inlinePatterns["implicit_pr"] = ImplicitPRPattern(
                    IMPLICIT_PR_RE
                )
                md.inlinePatterns["explicit_fork_issue"] = ExplicitLinkPattern(
                    EXPLICIT_LINK_RE
                )
                md.inlinePatterns["implicit_issue"] = ImplicitIssuePattern(
                    IMPLICIT_ISSUE_RE
                )
            md.inlinePatterns["striked"] = StrikeThroughPattern(
                STRIKE_THROUGH_RE
            )
            md.postprocessors[
                "encapsulate"
            ] = EncapsulateMarkdownPostprocessor()

        def _new_markdown_way():
            idx = md.inlinePatterns.get_index_for_name("autolink")
            md.inlinePatterns[idx].AUTOLINK_RE = AUTOLINK_RE

            # The number at the end is the priority, the highest priorities are
            # processed first.

            md.preprocessors.register(
                ImplicitIssuePreprocessor(), "implicit_issue", 100
            )
            md.inlinePatterns.register(
                MentionPattern(MENTION_RE), "mention", 95
            )
            # Customize the image linking to support lazy loading
            md.inlinePatterns.register(
                ImagePatternLazyLoad(
                    markdown.inlinepatterns.IMAGE_LINK_RE, md
                ),
                "image_link",
                90,
            )
            md.inlinePatterns.register(
                ImplicitCommitPattern(IMPLICIT_COMMIT_RE),
                "implicit_commit",
                85,
            )
            md.inlinePatterns.register(
                CommitLinkPattern(COMMIT_LINK_RE), "autolink2", 80
            )
            md.inlinePatterns.register(
                AutolinkPattern2(AUTOLINK_RE, md), "commit_links", 75
            )
            if pagure_config.get("ENABLE_TICKETS", True):
                md.inlinePatterns.register(
                    ImplicitPRPattern(IMPLICIT_PR_RE), "implicit_pr", 70
                )
                md.inlinePatterns.register(
                    ExplicitLinkPattern(EXPLICIT_LINK_RE),
                    "explicit_fork_issue",
                    65,
                )
                md.inlinePatterns.register(
                    ImplicitIssuePattern(IMPLICIT_ISSUE_RE),
                    "implicit_issue",
                    60,
                )
            md.inlinePatterns.register(
                StrikeThroughPattern(STRIKE_THROUGH_RE), "striked", 50
            )
            md.postprocessors.register(
                EncapsulateMarkdownPostprocessor(), "encapsulate", 100
            )

        if hasattr(md.inlinePatterns, "get_index_for_name"):
            _new_markdown_way()
        else:
            _old_mardkown_way()

        md.registerExtension(self)


def makeExtension(*arg, **kwargs):
    return PagureExtension(**kwargs)


def _issue_exists(user, namespace, repo, idx):
    """Utility method checking if a given issue exists."""

    repo_obj = pagure.lib.query.get_authorized_project(
        flask.g.session, project_name=repo, user=user, namespace=namespace
    )

    if not repo_obj:
        return False

    issue_obj = pagure.lib.query.search_issues(
        flask.g.session, repo=repo_obj, issueid=idx
    )
    if not issue_obj:
        return False

    return issue_obj


def _pr_exists(user, namespace, repo, idx):
    """Utility method checking if a given PR exists."""
    repo_obj = pagure.lib.query.get_authorized_project(
        flask.g.session, project_name=repo, user=user, namespace=namespace
    )

    if not repo_obj:
        return False

    pr_obj = pagure.lib.query.search_pull_requests(
        flask.g.session, project_id=repo_obj.id, requestid=idx
    )
    if not pr_obj:
        return False

    return pr_obj


def _commit_exists(user, namespace, repo, githash):
    """Utility method checking if a given commit exists."""
    repo_obj = pagure.lib.query.get_authorized_project(
        flask.g.session, project_name=repo, user=user, namespace=namespace
    )
    if not repo_obj:
        return False

    reponame = pagure.utils.get_repo_path(repo_obj)
    git_repo = pygit2.Repository(reponame)
    return githash in git_repo


def _obj_anchor_tag(user, namespace, repo, obj, text):
    """
    Utility method generating the link to an issue or a PR.

    :return: An element tree containing the href to the issue or PR
    :rtype:  xml.etree.ElementTree.Element
    """
    if isinstance(obj, six.string_types):
        url = flask.url_for(
            "ui_ns.view_commit",
            username=user,
            namespace=namespace,
            repo=repo,
            commitid=obj,
        )
        title = "Commit %s" % obj
    elif obj.isa == "issue":
        url = flask.url_for(
            "ui_ns.view_issue",
            username=user,
            namespace=namespace,
            repo=repo,
            issueid=obj.id,
        )
        if obj.private:
            title = "Private issue"
        else:
            if obj.status:
                title = "[%s] %s" % (obj.status, obj.title)
            else:
                title = obj.title
    else:
        url = flask.url_for(
            "ui_ns.request_pull",
            username=user,
            namespace=namespace,
            repo=repo,
            requestid=obj.id,
        )
        if obj.status:
            title = "[%s] %s" % (obj.status, obj.title)
        else:
            title = obj.title

    element = etree.Element("a")
    element.set("href", url)
    element.set("title", title)
    element.text = text
    return element


def _get_ns_repo_user():
    """Return the namespace, repo, user corresponding to the given request

    :return: A tuple of three string corresponding to namespace, repo, user
    :rtype: tuple(str, str, str)
    """

    root = flask.request.url_root
    url = flask.request.url

    user = flask.request.args.get("user") or None
    namespace = flask.request.args.get("namespace") or None
    repo = flask.request.args.get("repo") or None

    if not user and not repo:
        _log.debug("Extracting repo info from url: %s", url)
        if "fork/" in url:
            user, ext = url.split("fork/")[1].split("/", 1)
        else:
            ext = url.split(root)[1]

        if ext.count("/") >= 3:
            namespace, repo = ext.split("/", 2)[:2]
        else:
            repo = ext.split("/", 1)[0]

    return (namespace, repo, user)