RE-618: Automate C7->A8 happy path regression test on PR creation

.github/workflows/openstack.yml

@@ -0,0 +1,330 @@
+on:
+  push:
+    branches:
+      - "main"
+      - '!docs'
+    tags-ignore:
+      - "*"
+  pull_request:
+  workflow_dispatch:
+
+env:
+  TF_VAR_application_credential_id: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+  TF_VAR_application_credential_secret: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+  TF_VAR_ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
+  TF_VAR_ssh_public_key: ${{ secrets.SSH_PUBLIC_KEY }}
+  TF_VAR_os_auth_region: ${{ secrets.OS_AUTH_REGION }}
+  TF_VAR_os_auth_url: ${{ secrets.OS_AUTH_URL }}
+  TF_VAR_os_interface: ${{ secrets.OS_INTERFACE }}
+  TF_VAR_os_project_domain_name: ${{ secrets.OS_PROJECT_DOMAIN_NAME }}
+  TF_VAR_os_password: ${{ secrets.OS_PASSWORD }}
+  TF_VAR_os_project_id: ${{ secrets.OS_PROJECT_ID }}
+  TF_VAR_os_project_domain_id: ${{ secrets.OS_PROJECT_DOMAIN_ID }}
+  TF_VAR_os_project_name: ${{ secrets.OS_PROJECT_NAME }}
+  TF_VAR_os_region_name: ${{ secrets.OS_REGION_NAME }}
+  TF_VAR_os_username: ${{ secrets.OS_USERNAME }}
+  tf_working_directory: "./.github/workflows/openstack/terraform"
+
+jobs: 
+    terraform_openstack_create:
+        runs-on: self-hosted
+        defaults:
+          run:
+            working-directory: "./.github/workflows/openstack/terraform"
+        outputs:
+            VM_IP: ${{ steps.get_vm_ip.outputs.VM_IP }}
+        steps:
+        - uses: actions/checkout@v4
+        - name: Terraform fmt
+          id: fmt
+          run: |
+            terraform fmt -check
+          continue-on-error: true
+
+        - name: DEBUGVAR
+          id: debugvar
+          run: |
+            env | awk 'tolower($0)~/auth|project_name/'
+
+        - name: Terraform Init
+          id: init
+          run: |
+            terraform init
+
+        - name: Terraform Plan
+          id: plan
+          run: |
+            terraform plan -no-color
+
+        - name: Terraform Apply
+          id: apply
+          run: |
+            terraform apply -no-color -auto-approve
+
+        - name: Terraform Output File Create
+          run: |
+            terraform output -json > ${{ github.workspace }}/tf.out.json
+
+        - name: Get VM IP from Terraform Output File
+          id: get_vm_ip
+          run: |
+            echo "VM_IP=$(jq -r '.address.value' ${{ github.workspace }}/tf.out.json)" >> "$GITHUB_OUTPUT"
+            echo "$(jq -r '.address.value' ${{ github.workspace }}/tf.out.json)" > vm_ip
+
+        - name: Export VM_IP to Env
+          env:
+            VM_IP: ${{ steps.get_vm_ip.outputs.VM_IP }}
+          run: echo "The VM IP is $VM_IP"
+
+        - name: Upload JSON Output
+          uses: actions/upload-artifact@v4
+          with:
+            name: terraform_output_json
+            path: ${{ github.workspace }}/tf.out.json
+            overwrite: true
+
+    clone_elevate_repo:
+      runs-on: self-hosted
+      needs: terraform_openstack_create
+      outputs:
+        VM_IP: ${{ needs.terraform_openstack_create.outputs.VM_IP }}
+      steps:
+        - name: Checkout Repo and Commit
+          if: github.event_name != 'pull_request'
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.terraform_openstack_create.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            script: |
+              cd /opt
+              echo "[DEBUG]: ${{ github.ref }}"
+              echo "[DEBUG]: ${{ github.ref_name }}"
+              git clone --depth 1 --branch ${{ github.ref_name }} https://github.com/${{ github.repository }}.git
+              cd /opt/elevate
+              git status
+        - name: Checking out Repo and Commit
+          if: github.event_name == 'pull_request'
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.terraform_openstack_create.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            script: |
+              cd /opt
+              echo "[DEBUG]: ${{ github.ref }}"
+              echo "[DEBUG]: ${{ github.head_ref }}"
+              git clone --depth 1 --branch ${{ github.head_ref }} https://github.com/${{ github.repository }}.git
+              cd /opt/elevate
+              git status
+
+    setup_integration_checks:
+      runs-on: self-hosted
+      needs: clone_elevate_repo
+      outputs:
+        VM_IP: ${{ needs.clone_elevate_repo.outputs.VM_IP }}
+      steps:
+        - name: Setup for Integration Checks Prior to Running Elevate
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.clone_elevate_repo.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            script: |
+              chmod -v +x /opt/elevate/t/integration/setup
+              /opt/elevate/t/integration/setup
+
+    start_elevate:
+      runs-on: self-hosted
+      needs: setup_integration_checks
+      outputs:
+        VM_IP: ${{ needs.setup_integration_checks.outputs.VM_IP }}
+      steps:
+        - name: Starting Elevate
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.setup_integration_checks.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            script: |
+              cp -pv /opt/elevate/elevate-cpanel /scripts/elevate-cpanel
+              cp -pv /opt/elevate/.github/workflows/openstack/status_marker.sh /scripts/status_marker.sh
+              cp -pv /opt/elevate/.github/workflows/openstack/reboot_watch.sh /scripts/reboot_watch.sh
+              chmod -v +x /scripts/elevate-cpanel
+              /usr/local/cpanel/cpkeyclt
+              /scripts/elevate-cpanel --non-interactive --start &
+              /scripts/elevate-cpanel --log &
+              /scripts/elevate-cpanel --log | awk '/Rebooting into stage 2 of 5/ { print | "exit" }'
+
+    wait_for_stage_2_reboot:
+      runs-on: self-hosted
+      needs: start_elevate
+      outputs:
+        VM_IP: ${{ needs.start_elevate.outputs.VM_IP }}
+      steps:
+        - name: Wait For VM to Come Back From Stage 2 Reboot
+          working-directory: "./.github/workflows/openstack/"
+          run: |
+            ./ssh_retry.sh ${{ needs.start_elevate.outputs.VM_IP }}
+
+    watch_for_stage_3_reboot:
+      runs-on: self-hosted
+      needs: wait_for_stage_2_reboot
+      outputs:
+        VM_IP: ${{ needs.wait_for_stage_2_reboot.outputs.VM_IP }}
+      steps:
+        - name: Monitor Elevate for Stage 3 Reboot
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.wait_for_stage_2_reboot.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            timeout: 60m
+            command_timeout: 20m
+            debug: true
+            script: |
+              /scripts/status_marker.sh 3
+              /scripts/elevate-cpanel --log &
+              REGEX="Rebooting into stage 3 of 5" RETVAL=1 /scripts/reboot_watch.sh
+
+    wait_for_stage_3_reboot:
+      runs-on: self-hosted
+      needs: watch_for_stage_3_reboot
+      outputs:
+        VM_IP: ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP }}
+      steps:
+        - name: Wait For VM to Come Back From Stage 3 Reboot
+          working-directory: "./.github/workflows/openstack/"
+          run: |
+            ./ssh_retry.sh ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP }}
+
+    watch_for_stage_4_reboot:
+      runs-on: self-hosted
+      needs: wait_for_stage_3_reboot
+      outputs:
+        VM_IP: ${{ needs.wait_for_stage_3_reboot.outputs.VM_IP }}
+      steps:
+        - name: Monitor Elevate for Stage 4 Reboot
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.wait_for_stage_3_reboot.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            timeout: 30m
+            command_timeout: 20m
+            script: |
+              /scripts/status_marker.sh 4
+              /scripts/elevate-cpanel --log &
+              REGEX="Rebooting into stage 4 of 5" RETVAL=1 /scripts/reboot_watch.sh
+
+    wait_for_stage_4_reboot:
+      runs-on: self-hosted
+      needs: watch_for_stage_4_reboot
+      outputs:
+        VM_IP: ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP }}
+      steps:
+        - name: Wait For VM to Come Back From Stage 4 Reboot
+          working-directory: "./.github/workflows/openstack/"
+          run: |
+            ./ssh_retry.sh ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP }}
+
+    watch_for_stage_5_reboot:
+      runs-on: self-hosted
+      needs: wait_for_stage_4_reboot
+      outputs:
+        VM_IP: ${{ needs.wait_for_stage_4_reboot.outputs.VM_IP }}
+      steps:
+        - name: Monitor Elevate for Stage 5 Reboot
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.wait_for_stage_4_reboot.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            timeout: 30m
+            command_timeout: 20m
+            script: |
+              /scripts/status_marker.sh 5
+              /scripts/elevate-cpanel --log &
+              REGEX="Rebooting into stage 5 of 5" RETVAL=1 /scripts/reboot_watch.sh
+
+    wait_for_stage_5_reboot:
+      runs-on: self-hosted
+      needs: watch_for_stage_5_reboot
+      outputs:
+        VM_IP: ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }}
+      steps:
+        - name: Wait For VM to Come Back From Stage 5 Reboot
+          working-directory: "./.github/workflows/openstack/"
+          run: |
+            ./ssh_retry.sh ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }}
+
+    watch_for_final_reboot:
+      runs-on: self-hosted
+      needs: wait_for_stage_5_reboot
+      outputs:
+        VM_IP: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }}
+      steps:
+        - name: Watch Elevate for Final Reboot
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            timeout: 30m
+            command_timeout: 20m
+            script: |
+              /scripts/elevate-cpanel --log &
+              REGEX="Doing final reboot" RETVAL=1 /scripts/reboot_watch.sh
+
+    wait_for_final_reboot:
+      runs-on: self-hosted
+      needs: watch_for_final_reboot
+      outputs:
+        VM_IP: ${{ needs.watch_for_final_reboot.outputs.VM_IP }}
+      steps:
+        - name: Wait For VM to Come Back From Final Reboot
+          working-directory: "./.github/workflows/openstack/"
+          run: |
+            ./ssh_retry.sh ${{ needs.watch_for_final_reboot.outputs.VM_IP }}
+
+    verify_upgraded_os:
+      runs-on: self-hosted
+      needs: wait_for_final_reboot
+      outputs:
+        VM_IP: ${{ needs.wait_for_final_reboot.outputs.VM_IP }}
+      steps:
+        - name: Verify End Result Integration Tests
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.wait_for_final_reboot.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            timeout: 5m
+            command_timeout: 1m
+            script: |
+              /usr/local/cpanel/3rdparty/bin/prove -lvm /opt/elevate/t/integration/*.t
+
+    terraform_openstack_destroy:
+      runs-on: self-hosted
+      needs: verify_upgraded_os
+      defaults:
+        run:
+          working-directory: "./.github/workflows/openstack/terraform"
+      steps:
+      - name: Download Terraform Output JSON
+        uses: actions/download-artifact@v4
+        with:
+          name: terraform_output_json
+          path: ${{ github.workspace }}/
+      - name: Destroy OpenStack VM
+        run: terraform destroy -no-color -auto-approve 
+

.github/workflows/openstack/reboot_watch.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+while [ $RETVAL -ne 0 ]; do
+	grep "${REGEX}" /var/log/elevate-cpanel.log;
+	RETVAL=$?; 
+	[ $RETVAL -eq 0 ] && echo "## [INFO] SUCCESS: Reboot text found in /var/log/elevate-cpanel.log  ##" && exit 0;
+	RETRIES=$((RETRIES+1));
+	[ $RETVAL -ne 0 ] && echo "## [DEBUG]: Retrying Reboot REGEX Search: Attempt ${RETRIES} ...";
+	sleep 1;
+done

.github/workflows/openstack/ssh_retry.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+RETVAL=1;
+RETRIES=0;
+HOST=$1;
+PORT=$2
+PORT="${PORT:=22}";
+# Default 30 RETRIES usually one second apart
+RETRY=$3
+RETRY="${RETRY:=30}";
+
+while [ $RETVAL -ne 0 ];
+do
+   # We want to exit immediately after we actually connect to SSH on default port.
+   nc -z ${HOST} ${PORT}
+   RETVAL=$?;
+   [ $RETVAL -eq 0 ] && echo "## [INFO] SUCCESS: Connected to SSH on ${HOST} ##" && exit 0;
+   RETRIES=$((RETRIES+1));
+   [ $RETVAL -ne 0 ] && echo "## [DEBUG]: Retrying SSH Connect: Attempt ${RETRIES} ...";
+
+   if [ ${RETRIES} -ge ${RETRY} ];
+   then
+       echo "## [ERROR]: ssh_retry.sh: MAX_RETRIES has been reached.";
+       exit 1;
+   fi;
+   sleep 5;
+done

.github/workflows/openstack/status_marker.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+STAGE=$1;
+RELEASE_INFO=$(cat /etc/redhat-release);
+CPANEL_VERSION=$(cat /usr/local/cpanel/version);
+
+RC=$(echo ${RELEASE_INFO} | wc -c)
+
+
+
+for ((i=1; i<=${RC}+17; i++)); do echo -n "#"; done
+echo;
+echo "# STAGE: ${STAGE} of 5                                           #";
+echo "# OS Release: ${RELEASE_INFO}   #";
+echo "# cP Version: ${CPANEL_VERSION}                               #";
+for ((i=1; i<=${RC}+17; i++)); do echo -n "#"; done
+echo;

.github/workflows/openstack/terraform/cloud-config.yaml

@@ -0,0 +1,5 @@
+---
+    users:
+      - name: root
+        lock_passwd: false
+    disable_root: false