Provide a way to run the tests in containers using RPMs on F29 and C7

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

dev/containers/centos7-rpms

@@ -1,40 +0,0 @@
-FROM centos:7
-
-RUN yum -y install \
-    epel-release
-
-RUN yum -y install \
-    yum-utils \
-    rpm-build \
-    python-setuptools \
-    python-coverage \
-    python-nose \
-    python-mock \
-    python-docutils \
-    python-enum \
-    python-flask \
-    python2-flake8 \
-    redis \
-    which
-
-RUN cd /pagure && \
-    yum install --enablerepo=epel-testing -y \
-    `grep "Requires:" /pagure/files/pagure.spec | \
-    awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \
-    sed -e "s|%{python_pkgversion}||"` && \
-    yum clean all && \
-    localedef -i en_US -f UTF-8 en_US.UTF-8
-
-RUN \
-    sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \
-    sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \
-    sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \
-       /pagure/requirements.txt && \
-    cd /pagure && python setup.py build
-
-
-#ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
-WORKDIR /pagure
-ENTRYPOINT ["python2", "/pagure/runtests.py", "run", "-f", "--py2"]
-#ENTRYPOINT ["py.test", "-x", "tests/"]
-CMD []

dev/containers/centos7-rpms-py2

@@ -0,0 +1,53 @@
+FROM centos:7
+
+RUN yum -y install \
+    epel-release
+
+ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo
+
+RUN yum -y install \
+    yum-utils \
+    rpm-build \
+    python-setuptools \
+    python-coverage \
+    python-nose \
+    python-mock \
+    python-docutils \
+    python-enum \
+    python-flask \
+    python-flake8 \
+    redis \
+    which \
+    git \
+    repoSpanner \
+    repoSpanner-bridge
+
+RUN yum install -y https://kojipkgs.fedoraproject.org//work/tasks/5657/32535657/python-werkzeug-0.9.6-1.el7.noarch.rpm
+
+RUN cd / \
+    && git clone https://pagure.io/pagure.git \
+    && chmod +x /pagure/dev/containers/runtests_py2.sh
+
+# Install all the requirements from the spec file and replace the macro
+# %{python_pkgversion} by empty string which thus installs all the py2
+# version of the dependencies.
+RUN cd /pagure && \
+    yum install --enablerepo=epel-testing -y \
+    `rpmspec -q --requires /pagure/files/pagure.spec | \
+    sed -e "s|%{python_pkgversion}||"` && \
+    yum clean all && \
+    localedef -i en_US -f UTF-8 en_US.UTF-8
+
+# The old version of setuptools in CentOS7 does not support restrictions on
+# the requirements file, so drop them
+RUN \
+    sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \
+    sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \
+    sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \
+       /pagure/requirements.txt && \
+    cd /pagure && python setup.py build
+
+
+WORKDIR /pagure
+ENTRYPOINT ["/pagure/dev/containers/runtests_py2.sh"]
+CMD []

dev/containers/f29-rpms-py3

@@ -0,0 +1,31 @@
+FROM fedora:29
+
+RUN dnf -y install \
+    python3-setuptools \
+    python3-coverage \
+    python3-nose \
+    python3-mock \
+    python3-docutils \
+    python3-black \
+    python3-flake8 \
+    redis \
+    which \
+    git
+
+RUN cd / \
+    && git clone https://pagure.io/pagure.git \
+    && chmod +x /pagure/dev/containers/runtests_py3.sh
+
+# Install all the requirements from the spec file and replace the macro
+# %{python_pkgversion} by '3' which thus installs all the py3 version of
+# the dependencies.
+RUN dnf install -y `grep "Requires:" /pagure/files/pagure.spec | \
+    awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \
+    sed -e "s|%{python_pkgversion}|3|"` && \
+    dnf clean all && \
+    cd /pagure && python setup.py build
+
+
+WORKDIR /pagure
+ENTRYPOINT ["/pagure/dev/containers/runtests_py3.sh"]
+CMD []

dev/containers/runtests_py2.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+
+ls -l /
+
+echo "============== ENVIRONMENT ============="
+/usr/bin/env
+echo "============== END ENVIRONMENT ============="
+
+if [ -n "$REPO" -a -n "$BRANCH" ]; then
+git remote rm proposed || true
+git gc --auto
+git remote add proposed "$REPO"
+git fetch proposed
+git checkout origin/master
+git config --global user.email "you@example.com"
+git config --global user.name "Your Name"
+git merge --no-ff "proposed/$BRANCH" -m "Merge PR"
+
+echo "Running tests for branch $BRANCH of repo $REPO"
+echo "Last commits:"
+git log -2
+fi
+
+python2 /pagure/runtests.py run -f --py2

dev/containers/runtests_py3.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+
+ls -l /
+
+echo "============== ENVIRONMENT ============="
+/usr/bin/env
+echo "============== END ENVIRONMENT ============="
+
+if [ -n "$REPO" -a -n "$BRANCH" ]; then
+git remote rm proposed || true
+git gc --auto
+git remote add proposed "$REPO"
+git fetch proposed
+git checkout origin/master
+git config --global user.email "you@example.com"
+git config --global user.name "Your Name"
+git merge --no-ff "proposed/$BRANCH" -m "Merge PR"
+
+echo "Running tests for branch $BRANCH of repo $REPO"
+echo "Last commits:"
+git log -2
+fi
+
+python3 /pagure/runtests.py run -f --py3

dev/containers/test_env_template

@@ -1,15 +0,0 @@
-FROM $base_image
-
-$epel_pkg
-$infra_repo
-
-RUN  $pkg_mgr install -y $pkg_list \
-     && $pkg_mgr clean all \
-# for the hooks we need to have /usr/bin/python
-     && cp /usr/bin/python3 /usr/bin/python \
-# for the alembic test we need /usr/bin/alembic
-     && cp /usr/bin/alembic-3 /usr/bin/alembic
-
-WORKDIR /pagure
-ENTRYPOINT [ "/pagure/runtests.py", "run" ]
-CMD []

dev/run-tests-container.py

@@ -1,21 +1,8 @@
-#! /usr/bin/python3
+#! /usr/bin/env python
+
 import argparse
 import os
 import subprocess as sp
-from string import Template
-
-TEMPLATE = "dev/docker/test_env_template"
-
-PKG_LIST = "python3-alembic python3-arrow python3-binaryornot \
-            python3-bleach python3-blinker python3-chardet python3-cryptography \
-            python3-docutils python3-flask python3-fedora-flask \
-            python3-flask-wtf python3-bcrypt python3-jinja2 \
-            python3-markdown python3-munch python3-openid-cla \
-            python3-openid-teams python3-psutil python3-pygit2 python3-pillow \
-            python3-sqlalchemy python3-straight-plugin python3-wtforms \
-            python3-nose python3-coverage python3-mock python3-mock \
-            python3-eventlet python3-flask-oidc python3-flake8 python3-celery \
-            python3-redis python3-trololio python3-beautifulsoup4 python3-black redis vim git"
 
 
 def setup_parser():
@@ -56,69 +43,38 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     if args.centos is True:
-        base_image = "centos:7"
-        pkg_mgr = "yum"
-        epel_pkg = "RUN yum -y install epel-release"
-        infra_repo = (
-            "ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo"
-        )
-        container_name = "pagure-test-centos"
-        PKG_LIST += "python34 python34-coverage"
+        container_names = ["pagure-c7-rpms-py2"]
+        container_files = ["centos7-rpms-py2"]
+    elif args.fedora is True:
+        container_names = ["pagure-f29-rpms-py3"]
+        container_files = ["f29-rpms-py3"]
     else:
-        base_image = "registry.fedoraproject.org/fedora:latest"
-        pkg_mgr = "dnf"
-        container_name = "pagure-test-fedora"
-        epel_pkg = ""
-        infra_repo = ""
+        container_names = ["pagure-f29-rpms-py3", "pagure-c7-rpms-py2"]
+        container_files = ["f29-rpms-py3", "centos7-rpms-py2"]
 
-    with open(TEMPLATE, "r") as fp:
-        t = Template(fp.read())
-    with open("dev/docker/test_env", "w") as fp:
-        fp.write(
-            t.substitute(
-                base_image=base_image,
-                pkg_list=PKG_LIST,
-                pkg_mgr=pkg_mgr,
-                epel_pkg=epel_pkg,
-                infra_repo=infra_repo,
+    for idx, container_name in enumerate(container_names):
+        if args.skip_build is not False:
+            print("------ Building Container Image -----")
+            sp.call(
+                [
+                    "podman",
+                    "build",
+                    "--rm",
+                    "-t",
+                    container_name,
+                    "-f",
+                    "dev/containers/%s" % container_files[idx],
+                    "dev/containers",
+                ]
             )
-        )
-
-    if args.skip_build is not False:
-        print("------ Building Docker Image -----")
-        sp.run(
-            [
-                "podman",
-                "build",
-                "--rm",
-                "-t",
-                container_name,
-                "-f",
-                "dev/docker/test_env",
-                "dev/docker",
-            ]
-        )
-    if args.shell:
-        print("--------- Shelling in the container --------------")
-        command = [
-            "podman",
-            "run",
-            "-it",
-            "--rm",
-            "--name",
-            container_name,
-            "-v",
-            "{}:/pagure".format(os.getcwd()),
-            "--entrypoint=/bin/bash",
-            container_name,
-        ]
-        sp.run(command)
 
-    else:
+        result_path = "{}/results_{}".format(os.getcwd(), container_files[idx])
+        if not os.path.exists(result_path):
+            os.mkdir(result_path)
 
-        print("--------- Running Test --------------")
-        sp.run(
-            [
+        if args.shell:
+            print("--------- Shelling in the container --------------")
+            command = [
                 "podman",
                 "run",
                 "-it",
@@ -126,8 +82,34 @@ if __name__ == "__main__":
                 "--name",
                 container_name,
                 "-v",
-                "{}:/pagure".format(os.getcwd()),
+                "{}/results_{}:/pagure/results".format(
+                    os.getcwd(), container_files[idx]),
+                "-e",
+                "BRANCH=$BRANCH",
+                "-e",
+                "REPO=$REPO",
+                "--entrypoint=/bin/bash",
                 container_name,
-                args.test_case,
             ]
-        )
+            sp.call(command)
+        else:
+            print("--------- Running Test --------------")
+            sp.call(
+                [
+                    "podman",
+                    "run",
+                    "-it",
+                    "--rm",
+                    "--name",
+                    container_name,
+                    "-v",
+                    "{}/results_{}:/pagure/results".format(
+                        os.getcwd(), container_files[idx]),
+                    "-e",
+                    "BRANCH={}".format(os.environ.get("BRANCH") or ""),
+                    "-e",
+                    "REPO={}".format(os.environ.get("REPO") or ""),
+                    container_name,
+                    args.test_case,
+                ]
+            )

run_ci_tests_containers.sh

@@ -0,0 +1,38 @@
+set -x
+
+yum install -y podman
+
+sysctl -w fs.file-max=2048
+
+set -e
+
+echo $BRANCH $REPO
+
+podman build --rm -t pagure-f29-rpms-py3 \
+    -f dev/containers/f29-rpms-py3 \
+    dev/containers
+
+if [ ! -d `pwd`/results_f29-rpms-py3 ]; then
+  mkdir `pwd`/results_f29-rpms-py3;
+fi
+
+podman run --rm -it --name pagure-f29-rpms-py3 \
+    -v `pwd`/results_f29-rpms-py3:/pagure/results \
+    -e BRANCH=$BRANCH \
+    -e REPO=$REPO \
+    pagure-f29-rpms-py3
+
+
+podman build --rm -t pagure-c7-rpms-py2 \
+    -f dev/containers/centos7-rpms-py2 \
+    dev/containers
+
+if [ ! -d `pwd`/results_centos7-rpms-py2 ]; then
+  mkdir `pwd`/results_centos7-rpms-py2;
+fi
+
+podman run --rm -it --name pagure-c7-rpms-py2 \
+    -v `pwd`/results_centos7-rpms-py2:/pagure/results \
+    -e BRANCH=$BRANCH \
+    -e REPO=$REPO \
+    pagure-c7-rpms-py2