Initial attempt at native s390x builds

.github/actions/setup-vm-creds/action.yml

@@ -0,0 +1,39 @@
+name: Setup VM Credentials
+description: |
+  This action will setup the runner with the necessary credentials to create and
+  interact with VMs
+
+inputs:
+  gcp-ssh-key:
+    description:
+      The SSH private key to use for GCP
+  gcp-ssh-key-pub:
+    description:
+      The SSH public key to use for GCP
+  s390x-key:
+    description:
+      The SSH private key to use for s390x
+  ppc64le-key:
+    description:
+      The SSH private key to use for ppc64le
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+          mkdir -p "$HOME/.ssh"
+          chmod 0700 "$HOME/.ssh"
+
+          function copy_secret_to_file() {
+            local secret="$1"
+            local destination="$2"
+            local perms="$3"
+
+            echo "$secret" > "$destination"
+            chmod "$perms" "$destination"
+          }
+
+          copy_secret_to_file "${{ inputs.gcp-ssh-key }}" "$HOME/.ssh/GCP_SSH_KEY" 0600
+          copy_secret_to_file "${{ inputs.gcp-ssh-key-pub }}" "$HOME/.ssh/GCP_SSH_KEY.pub" 0600
+          copy_secret_to_file "${{ inputs.s390x-key }}" "$HOME/.ssh/acs-s390x-rsa.prv" 0600
+          copy_secret_to_file "${{ inputs.ppc64le-key }}" "$HOME/.ssh/acs-ppc64le-rsa.prv" 0600

.github/workflows/collector-builder.yml

@@ -37,13 +37,28 @@ jobs:
               - builder/Dockerfile
               - .github/workflows/collector-builder.yml
 
+  create-s390x-build-vms:
+    if: needs.builder-needs-rebuilding.outputs.build-image == 'true' ||
+      (github.event_name == 'push' && (
+        github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
+      )) ||
+      contains(github.event.pull_request.labels.*.name, 'build-builder-image')
+    needs:
+      - builder-needs-rebuilding
+    uses: ./.github/workflows/vms/create-vms.yml
+    with:
+      vm_type: rhel-s390x
+      job-tag: builder
+      kind: build
+
   build-builder-image:
     name: Build the builder image
     runs-on: ubuntu-latest
     # Multiarch builds sometimes take for eeeeeeeeeever
     timeout-minutes: 480
     needs:
     - builder-needs-rebuilding
+    - create-s390x-build-vms
     if: |
       needs.builder-needs-rebuilding.outputs.build-image == 'true' ||
       (github.event_name == 'push' && (
@@ -59,6 +74,8 @@ jobs:
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
+      BUILD_TYPE: ci
+      JOB_ID: ${{ needs.create-s390x-build-vms.outputs.job-id }}
 
     steps:
       - uses: actions/checkout@v3
@@ -94,22 +111,62 @@ jobs:
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
+      - uses: ./.github/actions/setup-vm-creds
+        with:
+          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
+          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
+          s390x-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
+          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
+
+
+      - name: Download inventory
+        if: matrix.arch == 's390x'
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ needs.create-s390x-build-vms.outputs.inventory }}
+
+      - name: Install ansible dependencies
+        run: |
+          ls -lah .
+          ls -lah ansible
+          ls -lah ansible/ci
+          ansible-galaxy install -r ansible/requirements.yml
+
       - name: Build images
         if: |
           github.event_name == 'push' ||
           matrix.arch == 'amd64' ||
           contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
         timeout-minutes: 480
         run: |
-          ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
             --connection local \
             -i localhost, \
             --limit localhost \
             -e arch='${{ matrix.arch }}' \
+            -e collector_builder_tag='${{ env.COLLECTOR_BUILDER_TAG }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
 
+      - name: Build s390x images
+        if: |
+          github.event_name == 'push' ||
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
+        timeout-minutes: 480
+        run: |
+          ansible-playbook \
+            -i ansible/ci \
+            -e arch='${{ matrix.arch }}' \
+            -e collector_builder_tag='${{ env.COLLECTOR_BUILDER_TAG }}' \
+            -e collector_git_ref='${{ github.sha }}' \
+            -e @'${{ github.workspace }}/ansible/secrets.yml' \
+            ansible/ci-build-builder.yml
+
+      - name: Destroy VMs
+        if: always()
+        run: |
+          make -C ansible destroy-vms
+
   create-multiarch-manifest:
     needs:
     - build-builder-image

.github/workflows/collector-slim.yml

@@ -25,16 +25,29 @@ env:
   RHACS_ENG_IMAGE: quay.io/rhacs-eng/collector:${{ inputs.collector-tag }}
 
 jobs:
+  create-s390x-build-vms:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') || github.event_name == 'push' }}
+    uses: ./.github/workflows/vms/create-vms.yml
+    with:
+      vm_type: rhel-s390x
+      job-tag: builder
+      kind: build
+
   build-collector-image:
     name: Build the collector slim image
     runs-on: ubuntu-latest
+    if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
+    needs:
+      - create-s390x-build-vms
     strategy:
       fail-fast: false
       matrix:
         arch: [amd64, ppc64le, s390x, arm64]
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
+      JOB_ID: ${{ needs.create-s390x-build-vms.outputs.job-id }}
+      BUILD_TYPE: ci
 
     steps:
       - uses: actions/checkout@v3
@@ -68,33 +81,78 @@ jobs:
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
+      - uses: ./.github/actions/setup-vm-creds
+        with:
+          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
+          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
+          s390x-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
+          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
+
+      - name: Download inventory
+        if: matrix.arch == 's390x'
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ needs.create-s390x-build-vms.outputs.inventory }}
+
+      - name: Install ansible dependencies
+        run: |
+          ls -lah .
+          ls -lah ansible
+          ls -lah ansible/ci
+          ansible-galaxy install -r ansible/requirements.yml
+
       - name: Build images
         if: |
           github.event_name == 'push' ||
           matrix.arch == 'amd64' ||
-          contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
         timeout-minutes: 480
         run: |
-          ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
             --connection local \
             -i localhost, \
             --limit localhost \
             -e collector_image='${{ inputs.collector-image }}' \
+            -e collector_tag='${{ inputs.collector-tag }}' \
+            -e collector_builder_tag='${{ env.COLLECTOR_BUILDER_TAG }}' \
             -e arch='${{ matrix.arch }}' \
             -e disable_profiling="${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }}" \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
         env:
           ANSIBLE_CONFIG: ansible/ansible.cfg
 
+      - name: Build s390x image
+        if: |
+          github.event_name == 'push' ||
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
+        timeout-minutes: 480
+        run: |
+          ansible-playbook \
+            -i ansible/ci \
+            -e collector_image='${{ inputs.collector-image }}' \
+            -e collector_tag='${{ inputs.collector-tag }}' \
+            -e collector_git_ref='${{ github.sha }}' \
+            -e collector_builder_tag='${{ env.COLLECTOR_BUILDER_TAG }}' \
+            -e arch='${{ matrix.arch }}' \
+            -e disable_profiling="true" \
+            -e @'${{ github.workspace }}/ansible/secrets.yml' \
+            ansible/ci-build-collector.yml
+        env:
+          ANSIBLE_CONFIG: ansible/ansible.cfg
+
       - name: Save CMake cache
         uses: actions/upload-artifact@v3
         with:
           name: cmake-${{ matrix.arch }}
           path: cmake-build/CMakeCache.txt
           retention-days: 1
 
+      - name: Destroy Build VMs
+        if: always()
+        run: |
+          make -C ansible destroy-vms
+
   create-multiarch-manifest:
     needs:
     - build-collector-image

.github/workflows/integration-tests-vm-type.yml

@@ -36,8 +36,16 @@ on:
         default: ""
 
 jobs:
-  run:
+  create-vms:
+    uses: ./.github/workflows/vms/create-vms.yml
+    with:
+      vm_type: ${{ inputs.vm_type }}
+      job-tag: ${{ inputs.job-tag }}
+      kind: test
+  tests:
     runs-on: ubuntu-latest
+    needs:
+      - create-vms
     env:
       #
       # JOB_ID is a little odd in that it needs to be unique per vm_type,
@@ -55,7 +63,7 @@ jobs:
       #
       # vm_type may contain hyphens, so the id is normalized below
       #
-      JOB_ID: ${{ github.run_id }}${{ inputs.vm_type }}${{ inputs.job-tag }}
+      JOB_ID: ${{ needs.create-vms.outputs.job-id }}
       GCP_SSH_KEY_FILE: ~/.ssh/GCP_SSH_KEY
       BUILD_TYPE: ci
       VM_TYPE: "${{ inputs.vm_type }}"
@@ -68,96 +76,22 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: '1.19' # to match the requirement in the integration tests
+
       - uses: actions/setup-python@v3
         with:
           python-version: "3.10"
 
-      - name: Install python dependencies
-        run: python -m pip install -r ansible/requirements.txt
-
-      - name: Install ansible dependencies
-        run: ansible-galaxy collection install -r ansible/ansible-collections.yml
-
-      - name: Authenticate with GCP
-        uses: 'google-github-actions/auth@v1'
+      - uses: ./.github/actions/setup-vm-creds
         with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
-
-      - name: Setup GCP
-        uses: 'google-github-actions/setup-gcloud@v1'
-
-      - name: Copy creds files
-        run: |
-          # Temporary compatibility with OSCI
-          mkdir -p /tmp/secret/stackrox-collector-e2e-tests
-          cp "$GOOGLE_APPLICATION_CREDENTIALS" /tmp/secret/stackrox-collector-e2e-tests/GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT
-
-          mkdir -p "$HOME/.ssh"
-          chmod 0700 "$HOME/.ssh"
-
-          function copy_secret_to_file() {
-            local secret="$1"
-            local destination="$2"
-            local perms="$3"
-
-            echo "$secret" > "$destination"
-            chmod "$perms" "$destination"
-          }
-
-          copy_secret_to_file "$GCP_SSH_KEY" "$HOME/.ssh/GCP_SSH_KEY" 0600
-          copy_secret_to_file "$GCP_SSH_KEY_PUB" "$HOME/.ssh/GCP_SSH_KEY.pub" 0600
-          copy_secret_to_file "$IBM_S390X_SSH_KEY" "$HOME/.ssh/acs-sshkey_rsa.prv" 0600
-
-          if [[ '${{ inputs.vm_type }}' =~ ppc64le ]]; then
-            copy_secret_to_file "$IBM_PPC64LE_SSH_KEY" "$HOME/.ssh/acs-sshkey_rsa.prv" 0600
-          fi
-        env:
-          GCP_SSH_KEY: ${{ secrets.GCP_SSH_KEY }}
-          GCP_SSH_KEY_PUB: ${{ secrets.GCP_SSH_KEY_PUB }}
-          IBM_S390X_SSH_KEY: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
-          IBM_PPC64LE_SSH_KEY: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
+          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
+          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
+          s390x-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
+          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
 
-      - name: Set environment
-        run: |
-          NORM_JOB_ID="${JOB_ID//-/}"
-          echo "JOB_ID=${NORM_JOB_ID}" | tee -a "$GITHUB_ENV"
-
-          if [[ '${{ contains(github.event.pull_request.labels.*.name, 'integration-tests-trace-logging') }}' == 'true' ]]; then
-            echo "COLLECTOR_LOG_LEVEL=trace" | tee -a "$GITHUB_ENV"
-          fi
-
-          if [[ '${{ inputs.vm_type }}' =~ s390x ]]; then
-            {
-              echo "IC_API_KEY=${{ secrets.IBM_CLOUD_S390X_API_KEY }}"
-              echo "IC_REGION=ca-tor"
-            } >> "${GITHUB_ENV}"
-          fi
-
-          if [[ '${{ inputs.vm_type }}' =~ ppc64le ]]; then
-            {
-              echo "IC_API_KEY=${{ secrets.IBM_CLOUD_POWER_API_KEY }}"
-              echo "IC_REGION=osa"
-              echo "IC_ZONE=osa21"
-              echo "IBM_CLOUD_POWER_SSH_PUBLIC_KEY=${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }}"
-            } >> "${GITHUB_ENV}"
-          fi
-
-          if [[ '${{ inputs.vm_type }}' =~ rhel ]]; then
-            {
-              echo "REDHAT_USERNAME=${{ secrets.REDHAT_USERNAME }}"
-              echo "REDHAT_PASSWORD=${{ secrets.REDHAT_PASSWORD }}"
-            } >> "${GITHUB_ENV}"
-          fi
-      #Added workaround for ssh connection issue with power vm from ubuntu machine.
-      #Changing mtu works in resolving the issue
-      - name: Set MTU for Power VMs
-        if: ${{ contains(inputs.vm_type, 'ppc64le') }}
-        run: sudo ifconfig eth0 mtu 1000 up
-
-      - name: Create VMs
-        if: ${{ ! inputs.run-benchmarks }}
-        run: |
-          make -C "${{ github.workspace }}/ansible" create-ci-vms
+      - name: Download inventory
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ needs.create-vms.outputs.inventory }}
 
       - name: Run Tests
         if: ${{ ! inputs.run-benchmarks }}