Fix test containers used on release-0.3

.github/workflows/ci.yml

@@ -72,6 +72,10 @@ jobs:
         rustup component add rustfmt
         make format-check
 
+  unit-tests:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
+
   vars:
     runs-on: ubuntu-24.04
     outputs:
@@ -180,18 +184,20 @@ jobs:
           podman manifest inspect ${{ needs.vars.outputs.rhacs-eng-image }}
           podman manifest push ${{ needs.vars.outputs.rhacs-eng-image }}
 
-  unit-tests:
-    uses: ./.github/workflows/unit-tests.yml
+  test-containers-build:
+    uses: ./.github/workflows/test-containers-build.yml
     secrets: inherit
 
   integration-tests:
     needs:
       - vars
       - manifest-stackrox-io
       - manifest-rhacs-eng
+      - test-containers-build
     uses: ./.github/workflows/integration-tests.yml
     with:
       tag: ${{ needs.vars.outputs.tag }}
+      qa-tag: ${{ needs.test-containers-build.outputs.qa-tag }}
     secrets: inherit
 
   performance-tests:

.github/workflows/integration-tests.yml

@@ -13,6 +13,9 @@ on:
       tag:
         description: The container image tag to test
         type: string
+      qa-tag:
+        description: The test containers tag to use
+        type: string
       job-tag:
         description: Additional tag to prevent collision on GCP VM naming
         type: string
@@ -68,6 +71,7 @@ jobs:
       env:
         FACT_VERSION: ${{ inputs.version }}
         FACT_REGISTRY: ${{ inputs.registry }}
+        FACT_QA_TAG: ${{ inputs.qa-tag }}
         FACT_TAG: ${{ inputs.tag }}
       run: |
         FACT_IMAGE_NAME="$(make -sC "${GITHUB_WORKSPACE}/fact" image-name)"
@@ -77,6 +81,7 @@ jobs:
         fact:
           image: ${FACT_IMAGE_NAME}
           version: ${FACT_VERSION}
+        qa_tag: ${FACT_QA_TAG}
         quay:
           username: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
           password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}

.github/workflows/konflux-tests.yml

@@ -40,14 +40,20 @@ jobs:
           image: rhacs-eng/release-fact:${{ needs.init.outputs.fact-tag }}
           limit: 2400 # 40m
 
+  test-containers-build:
+    uses: ./.github/workflows/test-containers-build.yml
+    secrets: inherit
+
   run-konflux-tests:
     uses: ./.github/workflows/integration-tests.yml
     needs:
     - init
     - wait-for-images
+    - test-containers-build
     with:
       version: ${{ github.head_ref || github.ref_name }}
       registry: quay.io/rhacs-eng/release-fact
       tag: ${{ needs.init.outputs.fact-tag }}
+      qa-tag: ${{ needs.test-containers-build.outputs.qa-tag }}
       job-tag: konf
     secrets: inherit

.github/workflows/test-containers-build.yml

@@ -0,0 +1,126 @@
+name: Build integration test containers
+
+on:
+  workflow_call:
+    outputs:
+      qa-tag:
+        value: ${{ jobs.determine-qa-tag.outputs.qa-tag }}
+
+jobs:
+  should-build-test-containers:
+    name: Determine if tests image needs to be built
+    runs-on: ubuntu-24.04
+    outputs:
+      build-image: ${{ (github.event_name != 'schedule' &&
+                        (steps.changed.outputs.qa-tag == 'true' ||
+                         contains(github.event.pull_request.labels.*.name, 'rebuild-test-container'))) }}
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: dorny/paths-filter@v4
+        id: changed
+        with:
+          list-files: shell
+          base: ${{ github.ref }}
+
+          # Only trigger a rebuild when the QA tag has changed
+          filters: |
+            qa-tag:
+              - tests/containers/QA_TAG
+
+  determine-qa-tag:
+    name: Determine QA tag
+    runs-on: ubuntu-24.04
+    needs:
+      - should-build-test-containers
+    outputs:
+      qa-tag: ${{ steps.qa-tag.outputs.qa-tag }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: true
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Determine QA tag
+        id: qa-tag
+        run: |
+          QA_TAG="$(cat "${GITHUB_WORKSPACE}/tests/containers/QA_TAG")"
+          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" && "${{ needs.should-build-test-containers.outputs.build-image }}" == "true" ]]; then
+            QA_TAG="${QA_TAG}-$(make tag)"
+          fi
+          echo "::notice::Using QA tag: ${QA_TAG}"
+          echo "qa-tag=${QA_TAG}" >> "$GITHUB_OUTPUT"
+
+  build-test-containers:
+    name: Build and push test containers
+    runs-on: ubuntu-24.04
+    needs:
+      - determine-qa-tag
+      - should-build-test-containers
+    if: needs.should-build-test-containers.outputs.build-image == 'true' ||
+        contains(github.event.pull_request.labels.*.name, 'rebuild-test-container')
+    env:
+      FACT_QA_TAG: ${{ needs.determine-qa-tag.outputs.qa-tag }}
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to quay.io/rhacs-eng
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
+          password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
+
+      - name: Check images already exist
+        id: images-exist
+        run: |
+          manifest_exists() {
+            local image=$1 output
+            if output=$(docker manifest inspect "${image}" 2>&1); then
+              return 0
+            elif echo "${output}" | grep -Eiq 'no such manifest:|manifest unknown'; then
+              return 1
+            else
+              printf '%s: %s\n' "${image}" "${output}" >&2
+              return 2
+            fi
+          }
+
+          image_names=$(make --no-print-directory -s \
+            -C "${GITHUB_WORKSPACE}/tests/containers" image-names) || exit $?
+
+          EXIST=true
+          while IFS= read -r image_name; do
+            [[ -z "${image_name}" ]] && continue
+            if manifest_exists "${image_name}"; then
+              continue
+            else
+              status=$?
+            fi
+
+            if [[ "${status}" -eq 1 ]]; then
+              EXIST=false
+            else
+              exit "${status}"
+            fi
+          done <<< "${image_names}"
+
+          if [[ "${EXIST}" == false ]]; then
+            echo "::notice::At least one test container manifest is absent"
+          fi
+          echo "exist=${EXIST}" >> "${GITHUB_OUTPUT}"
+
+      - name: Build and Push containers
+        if: steps.images-exist.outputs.exist == 'false' || contains(github.event.pull_request.labels.*.name, 'rebuild-test-container')
+        env:
+          PLATFORM: linux/amd64,linux/arm64
+        run: |
+          make -C "${GITHUB_WORKSPACE}/tests/containers" build-and-push

ansible/run-tests.yml

@@ -37,6 +37,8 @@
 
     - block:
       - name: Run tests
+        environment:
+          FACT_QA_TAG: "{{ qa_tag | default(None) }}"
         ansible.builtin.shell:
           cmd: |
             cd "${HOME}/fact"

tests/conftest.py

@@ -228,3 +228,8 @@ def fact(request, docker_client, fact_config, server, logs_dir, test_file):
 def pytest_addoption(parser):
     parser.addoption('--image', action='store', default='quay.io/stackrox-io/fact:latest',
                      help='The image to be used for testing')
+    parser.addoption(
+        '--no-local-builds',
+        action='store_true',
+        help='Do not build test containers locally',
+    )

tests/containers.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import os
+
+import docker
+import docker.errors
+from docker.models.images import Image
+
+
+def read_qa_tag() -> str:
+    tag = os.environ.get('FACT_QA_TAG')
+    if tag is not None:
+        return tag
+
+    with open('containers/QA_TAG') as f:
+        return f.read().strip()
+
+
+QA_TAG = read_qa_tag()
+QA_REPOSITORY = os.environ.get(
+    'FACT_QA_REPOSITORY', 'quay.io/rhacs-eng/qa-multi-arch'
+)
+
+
+def pull_or_build(
+    docker_client: docker.DockerClient,
+    tag: str,
+    path: str | None,
+    no_local_builds: bool,
+    containerfile: str = 'Containerfile',
+) -> Image:
+    tag = f'{tag}-{QA_TAG}'
+    image = f'{QA_REPOSITORY}:{tag}'
+
+    try:
+        return docker_client.images.get(image)
+    except docker.errors.ImageNotFound:
+        print(f'{image} not found locally, attempting to pull')
+
+    try:
+        return docker_client.images.pull(QA_REPOSITORY, tag=tag)
+    except docker.errors.APIError as e:
+        if no_local_builds:
+            raise e
+
+        if e.status_code != 401 and e.status_code != 404:
+            raise e
+        print(f'Failed to pull image: {e}')
+        print('Attempting to build image from source')
+
+        image, _ = docker_client.images.build(
+            path=path, tag=image, dockerfile=containerfile
+        )
+        return image

tests/containers/Makefile

@@ -0,0 +1,10 @@
+include constants.mk
+
+TARGETS := build build-and-push image-names
+DIRS := $(wildcard ./*/)
+
+$(TARGETS): $(DIRS)
+$(DIRS):
+	$(MAKE) -C $@ $(MAKECMDGOALS)
+
+.PHONY: $(TARGETS) $(DIRS)

tests/containers/QA_TAG

@@ -0,0 +1 @@
+0.2.0

tests/containers/base/Containerfile

@@ -0,0 +1,2 @@
+# CentOS Stream 10
+FROM quay.io/centos/centos@sha256:6f8d618844ed5af4365bad24bea19fb0435fd5ec808201af9aeb19d7a28eb515

tests/containers/base/Makefile

@@ -0,0 +1,22 @@
+include ../constants.mk
+
+FACT_QA_BASE_TAG := fact-base-$(FACT_QA_TAG)
+
+all: build
+
+run:
+	$(DOCKER) buildx build $(BUILD_ARGS) --platform $(PLATFORM) \
+		-t quay.io/rhacs-eng/qa-multi-arch:$(FACT_QA_BASE_TAG) \
+		-f Containerfile \
+		$(CURDIR)
+
+build: BUILD_ARGS = --load
+build: run
+
+build-and-push: BUILD_ARGS = --push
+build-and-push: run
+
+image-names:
+	@echo quay.io/rhacs-eng/qa-multi-arch:$(FACT_QA_BASE_TAG)
+
+.PHONY: build build-and-push all run image-names

tests/containers/constants.mk

@@ -0,0 +1,13 @@
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+
+include $(REPO_ROOT)/constants.mk
+
+# PLATFORM can be a comma separated list of platforms of the form linux/<arch>
+# it is passed to buildx, which will build for each one (this is important
+# for pushing multi-arch images because `buildx --push` will push a manifest list
+# containing all the relevant platforms)
+PLATFORM ?= linux/amd64
+
+# Path needs to be absolute so subdirectories don't fail when they include this
+# file.
+FACT_QA_TAG ?= $(shell cat "$(REPO_ROOT)/tests/containers/QA_TAG")

tests/containers/editors/Containerfile

@@ -1,5 +1,9 @@
-FROM quay.io/fedora/fedora:43
+# CentOS Stream 10
+FROM quay.io/centos/centos@sha256:6f8d618844ed5af4365bad24bea19fb0435fd5ec808201af9aeb19d7a28eb515
 
-RUN dnf install -y \
-    neovim \
-    vim
+RUN dnf config-manager --set-enabled crb && \
+    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm && \
+    dnf install -y \
+    neovim-0.10.1-4.el10_0 \
+    vim-enhanced-2:9.1.083-25.el10 && \
+    dnf clean all

tests/containers/editors/Makefile

@@ -0,0 +1,22 @@
+include ../constants.mk
+
+FACT_QA_EDITORS_TAG := fact-editors-$(FACT_QA_TAG)
+
+all: build
+
+run:
+	$(DOCKER) buildx build $(BUILD_ARGS) --platform $(PLATFORM) \
+		-t quay.io/rhacs-eng/qa-multi-arch:$(FACT_QA_EDITORS_TAG) \
+		-f Containerfile \
+		$(CURDIR)
+
+build: BUILD_ARGS = --load
+build: run
+
+build-and-push: BUILD_ARGS = --push
+build-and-push: run
+
+image-names:
+	@echo quay.io/rhacs-eng/qa-multi-arch:$(FACT_QA_EDITORS_TAG)
+
+.PHONY: build build-and-push all run image-names

tests/containers/self-deleter/Makefile

@@ -0,0 +1,22 @@
+include ../constants.mk
+
+FACT_QA_SELF_DELETER_TAG := fact-self-deleter-$(FACT_QA_TAG)
+
+all: build
+
+run:
+	$(DOCKER) buildx build $(BUILD_ARGS) --platform $(PLATFORM) \
+		-t quay.io/rhacs-eng/qa-multi-arch:$(FACT_QA_SELF_DELETER_TAG) \
+		-f Containerfile \
+		$(CURDIR)
+
+build: BUILD_ARGS = --load
+build: run
+
+build-and-push: BUILD_ARGS = --push
+build-and-push: run
+
+image-names:
+	@echo quay.io/rhacs-eng/qa-multi-arch:$(FACT_QA_SELF_DELETER_TAG)
+
+.PHONY: build build-and-push all run image-names