From 9f12b6bf3753499f4306d0495cd7c8f2ad3693d5 Mon Sep 17 00:00:00 2001 From: Dave Lanning Date: Mon, 19 Aug 2024 17:27:02 +0000 Subject: [PATCH] RE-618: Add openstack actions workflow for Pull Requestsand pushes to main. --- .github/workflows/openstack.yml | 330 ++++++++++++++++++ .github/workflows/openstack/reboot_watch.sh | 10 + .github/workflows/openstack/ssh_retry.sh | 27 ++ .github/workflows/openstack/status_marker.sh | 17 + .../openstack/terraform/cloud-config.yaml | 5 + .../openstack/terraform/cloud-init.yml | 6 + .github/workflows/openstack/terraform/main.tf | 82 +++++ .../workflows/openstack/terraform/outputs.tf | 7 + .../openstack/terraform/variables.tf | 55 +++ t/integration/complete.t | 23 ++ 10 files changed, 562 insertions(+) create mode 100644 .github/workflows/openstack.yml create mode 100755 .github/workflows/openstack/reboot_watch.sh create mode 100755 .github/workflows/openstack/ssh_retry.sh create mode 100755 .github/workflows/openstack/status_marker.sh create mode 100644 .github/workflows/openstack/terraform/cloud-config.yaml create mode 100644 .github/workflows/openstack/terraform/cloud-init.yml create mode 100644 .github/workflows/openstack/terraform/main.tf create mode 100644 .github/workflows/openstack/terraform/outputs.tf create mode 100644 .github/workflows/openstack/terraform/variables.tf create mode 100644 t/integration/complete.t diff --git a/.github/workflows/openstack.yml b/.github/workflows/openstack.yml new file mode 100644 index 00000000..03f0be9a --- /dev/null +++ b/.github/workflows/openstack.yml @@ -0,0 +1,330 @@ +on: + push: + branches: + - "main" + - '!docs' + tags-ignore: + - "*" + pull_request: + workflow_dispatch: + +env: + TF_VAR_application_credential_id: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + TF_VAR_application_credential_secret: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + TF_VAR_ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }} + TF_VAR_ssh_public_key: ${{ secrets.SSH_PUBLIC_KEY }} + TF_VAR_os_auth_region: ${{ secrets.OS_AUTH_REGION }} + TF_VAR_os_auth_url: ${{ secrets.OS_AUTH_URL }} + TF_VAR_os_interface: ${{ secrets.OS_INTERFACE }} + TF_VAR_os_project_domain_name: ${{ secrets.OS_PROJECT_DOMAIN_NAME }} + TF_VAR_os_password: ${{ secrets.OS_PASSWORD }} + TF_VAR_os_project_id: ${{ secrets.OS_PROJECT_ID }} + TF_VAR_os_project_domain_id: ${{ secrets.OS_PROJECT_DOMAIN_ID }} + TF_VAR_os_project_name: ${{ secrets.OS_PROJECT_NAME }} + TF_VAR_os_region_name: ${{ secrets.OS_REGION_NAME }} + TF_VAR_os_username: ${{ secrets.OS_USERNAME }} + tf_working_directory: "./.github/workflows/openstack/terraform" + +jobs: + terraform_openstack_create: + runs-on: self-hosted + defaults: + run: + working-directory: "./.github/workflows/openstack/terraform" + outputs: + VM_IP: ${{ steps.get_vm_ip.outputs.VM_IP }} + steps: + - uses: actions/checkout@v4 + - name: Terraform fmt + id: fmt + run: | + terraform fmt -check + continue-on-error: true + + - name: DEBUGVAR + id: debugvar + run: | + env | awk 'tolower($0)~/auth|project_name/' + + - name: Terraform Init + id: init + run: | + terraform init + + - name: Terraform Plan + id: plan + run: | + terraform plan -no-color + + - name: Terraform Apply + id: apply + run: | + terraform apply -no-color -auto-approve + + - name: Terraform Output File Create + run: | + terraform output -json > ${{ github.workspace }}/tf.out.json + + - name: Get VM IP from Terraform Output File + id: get_vm_ip + run: | + echo "VM_IP=$(jq -r '.address.value' ${{ github.workspace }}/tf.out.json)" >> "$GITHUB_OUTPUT" + echo "$(jq -r '.address.value' ${{ github.workspace }}/tf.out.json)" > vm_ip + + - name: Export VM_IP to Env + env: + VM_IP: ${{ steps.get_vm_ip.outputs.VM_IP }} + run: echo "The VM IP is $VM_IP" + + - name: Upload JSON Output + uses: actions/upload-artifact@v4 + with: + name: terraform_output_json + path: ${{ github.workspace }}/tf.out.json + overwrite: true + + clone_elevate_repo: + runs-on: self-hosted + needs: terraform_openstack_create + outputs: + VM_IP: ${{ needs.terraform_openstack_create.outputs.VM_IP }} + steps: + - name: Checkout Repo and Commit + if: github.event_name != 'pull_request' + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.terraform_openstack_create.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + script: | + cd /opt + echo "[DEBUG]: ${{ github.ref }}" + echo "[DEBUG]: ${{ github.ref_name }}" + git clone --depth 1 --branch ${{ github.ref_name }} https://github.com/${{ github.repository }}.git + cd /opt/elevate + git status + - name: Checking out Repo and Commit + if: github.event_name == 'pull_request' + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.terraform_openstack_create.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + script: | + cd /opt + echo "[DEBUG]: ${{ github.ref }}" + echo "[DEBUG]: ${{ github.head_ref }}" + git clone --depth 1 --branch ${{ github.head_ref }} https://github.com/${{ github.repository }}.git + cd /opt/elevate + git status + + setup_integration_checks: + runs-on: self-hosted + needs: clone_elevate_repo + outputs: + VM_IP: ${{ needs.clone_elevate_repo.outputs.VM_IP }} + steps: + - name: Setup for Integration Checks Prior to Running Elevate + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.clone_elevate_repo.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + script: | + chmod -v +x /opt/elevate/t/integration/setup + /opt/elevate/t/integration/setup + + start_elevate: + runs-on: self-hosted + needs: setup_integration_checks + outputs: + VM_IP: ${{ needs.setup_integration_checks.outputs.VM_IP }} + steps: + - name: Starting Elevate + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.setup_integration_checks.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + script: | + cp -pv /opt/elevate/elevate-cpanel /scripts/elevate-cpanel + cp -pv /opt/elevate/.github/workflows/openstack/status_marker.sh /scripts/status_marker.sh + cp -pv /opt/elevate/.github/workflows/openstack/reboot_watch.sh /scripts/reboot_watch.sh + chmod -v +x /scripts/elevate-cpanel + /usr/local/cpanel/cpkeyclt + /scripts/elevate-cpanel --non-interactive --start & + /scripts/elevate-cpanel --log & + /scripts/elevate-cpanel --log | awk '/Rebooting into stage 2 of 5/ { print | "exit" }' + + wait_for_stage_2_reboot: + runs-on: self-hosted + needs: start_elevate + outputs: + VM_IP: ${{ needs.start_elevate.outputs.VM_IP }} + steps: + - name: Wait For VM to Come Back From Stage 2 Reboot + working-directory: "./.github/workflows/openstack/" + run: | + ./ssh_retry.sh ${{ needs.start_elevate.outputs.VM_IP }} + + watch_for_stage_3_reboot: + runs-on: self-hosted + needs: wait_for_stage_2_reboot + outputs: + VM_IP: ${{ needs.wait_for_stage_2_reboot.outputs.VM_IP }} + steps: + - name: Monitor Elevate for Stage 3 Reboot + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.wait_for_stage_2_reboot.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + timeout: 60m + command_timeout: 20m + debug: true + script: | + /scripts/status_marker.sh 3 + /scripts/elevate-cpanel --log & + REGEX="Rebooting into stage 3 of 5" RETVAL=1 /scripts/reboot_watch.sh + + wait_for_stage_3_reboot: + runs-on: self-hosted + needs: watch_for_stage_3_reboot + outputs: + VM_IP: ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP }} + steps: + - name: Wait For VM to Come Back From Stage 3 Reboot + working-directory: "./.github/workflows/openstack/" + run: | + ./ssh_retry.sh ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP }} + + watch_for_stage_4_reboot: + runs-on: self-hosted + needs: wait_for_stage_3_reboot + outputs: + VM_IP: ${{ needs.wait_for_stage_3_reboot.outputs.VM_IP }} + steps: + - name: Monitor Elevate for Stage 4 Reboot + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.wait_for_stage_3_reboot.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + timeout: 30m + command_timeout: 20m + script: | + /scripts/status_marker.sh 4 + /scripts/elevate-cpanel --log & + REGEX="Rebooting into stage 4 of 5" RETVAL=1 /scripts/reboot_watch.sh + + wait_for_stage_4_reboot: + runs-on: self-hosted + needs: watch_for_stage_4_reboot + outputs: + VM_IP: ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP }} + steps: + - name: Wait For VM to Come Back From Stage 4 Reboot + working-directory: "./.github/workflows/openstack/" + run: | + ./ssh_retry.sh ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP }} + + watch_for_stage_5_reboot: + runs-on: self-hosted + needs: wait_for_stage_4_reboot + outputs: + VM_IP: ${{ needs.wait_for_stage_4_reboot.outputs.VM_IP }} + steps: + - name: Monitor Elevate for Stage 5 Reboot + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.wait_for_stage_4_reboot.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + timeout: 30m + command_timeout: 20m + script: | + /scripts/status_marker.sh 5 + /scripts/elevate-cpanel --log & + REGEX="Rebooting into stage 5 of 5" RETVAL=1 /scripts/reboot_watch.sh + + wait_for_stage_5_reboot: + runs-on: self-hosted + needs: watch_for_stage_5_reboot + outputs: + VM_IP: ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }} + steps: + - name: Wait For VM to Come Back From Stage 5 Reboot + working-directory: "./.github/workflows/openstack/" + run: | + ./ssh_retry.sh ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }} + + watch_for_final_reboot: + runs-on: self-hosted + needs: wait_for_stage_5_reboot + outputs: + VM_IP: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }} + steps: + - name: Watch Elevate for Final Reboot + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + timeout: 30m + command_timeout: 20m + script: | + /scripts/elevate-cpanel --log & + REGEX="Doing final reboot" RETVAL=1 /scripts/reboot_watch.sh + + wait_for_final_reboot: + runs-on: self-hosted + needs: watch_for_final_reboot + outputs: + VM_IP: ${{ needs.watch_for_final_reboot.outputs.VM_IP }} + steps: + - name: Wait For VM to Come Back From Final Reboot + working-directory: "./.github/workflows/openstack/" + run: | + ./ssh_retry.sh ${{ needs.watch_for_final_reboot.outputs.VM_IP }} + + verify_upgraded_os: + runs-on: self-hosted + needs: wait_for_final_reboot + outputs: + VM_IP: ${{ needs.wait_for_final_reboot.outputs.VM_IP }} + steps: + - name: Verify End Result Integration Tests + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ needs.wait_for_final_reboot.outputs.VM_IP }} + username: 'root' + key: ${{ secrets.SSH_PRIVATE_KEY }} + port: '22' + timeout: 5m + command_timeout: 1m + script: | + /usr/local/cpanel/3rdparty/bin/prove -lvm /opt/elevate/t/integration/*.t + + terraform_openstack_destroy: + runs-on: self-hosted + needs: verify_upgraded_os + defaults: + run: + working-directory: "./.github/workflows/openstack/terraform" + steps: + - name: Download Terraform Output JSON + uses: actions/download-artifact@v4 + with: + name: terraform_output_json + path: ${{ github.workspace }}/ + - name: Destroy OpenStack VM + run: terraform destroy -no-color -auto-approve + diff --git a/.github/workflows/openstack/reboot_watch.sh b/.github/workflows/openstack/reboot_watch.sh new file mode 100755 index 00000000..838ca35f --- /dev/null +++ b/.github/workflows/openstack/reboot_watch.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +while [ $RETVAL -ne 0 ]; do + grep "${REGEX}" /var/log/elevate-cpanel.log; + RETVAL=$?; + [ $RETVAL -eq 0 ] && echo "## [INFO] SUCCESS: Reboot text found in /var/log/elevate-cpanel.log ##" && exit 0; + RETRIES=$((RETRIES+1)); + [ $RETVAL -ne 0 ] && echo "## [DEBUG]: Retrying Reboot REGEX Search: Attempt ${RETRIES} ..."; + sleep 1; +done diff --git a/.github/workflows/openstack/ssh_retry.sh b/.github/workflows/openstack/ssh_retry.sh new file mode 100755 index 00000000..4f5c0680 --- /dev/null +++ b/.github/workflows/openstack/ssh_retry.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +RETVAL=1; +RETRIES=0; +HOST=$1; +PORT=$2 +PORT="${PORT:=22}"; +# Default 30 RETRIES usually one second apart +RETRY=$3 +RETRY="${RETRY:=30}"; + +while [ $RETVAL -ne 0 ]; +do + # We want to exit immediately after we actually connect to SSH on default port. + nc -z ${HOST} ${PORT} + RETVAL=$?; + [ $RETVAL -eq 0 ] && echo "## [INFO] SUCCESS: Connected to SSH on ${HOST} ##" && exit 0; + RETRIES=$((RETRIES+1)); + [ $RETVAL -ne 0 ] && echo "## [DEBUG]: Retrying SSH Connect: Attempt ${RETRIES} ..."; + + if [ ${RETRIES} -ge ${RETRY} ]; + then + echo "## [ERROR]: ssh_retry.sh: MAX_RETRIES has been reached."; + exit 1; + fi; + sleep 5; +done diff --git a/.github/workflows/openstack/status_marker.sh b/.github/workflows/openstack/status_marker.sh new file mode 100755 index 00000000..8246705f --- /dev/null +++ b/.github/workflows/openstack/status_marker.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +STAGE=$1; +RELEASE_INFO=$(cat /etc/redhat-release); +CPANEL_VERSION=$(cat /usr/local/cpanel/version); + +RC=$(echo ${RELEASE_INFO} | wc -c) + + + +for ((i=1; i<=${RC}+17; i++)); do echo -n "#"; done +echo; +echo "# STAGE: ${STAGE} of 5 #"; +echo "# OS Release: ${RELEASE_INFO} #"; +echo "# cP Version: ${CPANEL_VERSION} #"; +for ((i=1; i<=${RC}+17; i++)); do echo -n "#"; done +echo; diff --git a/.github/workflows/openstack/terraform/cloud-config.yaml b/.github/workflows/openstack/terraform/cloud-config.yaml new file mode 100644 index 00000000..0445e91b --- /dev/null +++ b/.github/workflows/openstack/terraform/cloud-config.yaml @@ -0,0 +1,5 @@ +--- + users: + - name: root + lock_passwd: false + disable_root: false \ No newline at end of file diff --git a/.github/workflows/openstack/terraform/cloud-init.yml b/.github/workflows/openstack/terraform/cloud-init.yml new file mode 100644 index 00000000..1602682e --- /dev/null +++ b/.github/workflows/openstack/terraform/cloud-init.yml @@ -0,0 +1,6 @@ +#cloud-config +disable_root: false +ssh_pwauth: true +hostname: host +fqdn: host.domain.tbd +prefer_fqdn_over_hostname: true diff --git a/.github/workflows/openstack/terraform/main.tf b/.github/workflows/openstack/terraform/main.tf new file mode 100644 index 00000000..22f1cedd --- /dev/null +++ b/.github/workflows/openstack/terraform/main.tf @@ -0,0 +1,82 @@ +# Define required providers +terraform { + required_version = ">= 0.14.0" + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "~> 1.54.1" + } + } +} + +# Configure the OpenStack Provider +provider "openstack" { + user_name = var.user + application_credential_id = var.application_credential_id + application_credential_secret = var.application_credential_secret + auth_url = "https://keystone.hou-01.cloud.prod.cpanel.net:5000/v3" + region = var.os_auth_region +} + +data "openstack_images_image_ids_v2" "images" { + name_regex = var.image_name + sort = "updated_at" +} + +data "template_cloudinit_config" "config" { + gzip = true + base64_encode = true + + part { + content_type = "text/cloud-config" + content = "cloud-config.yaml" + } +} + +resource "tls_private_key" "ssh" { + algorithm = "ECDSA" + ecdsa_curve = "P384" +} + +resource "random_string" "keyname" { + length = 22 + special = false +} + +resource "openstack_compute_keypair_v2" "tf_remote_key" { + name = "${random_string.keyname.result}-deletethis" + public_key = tls_private_key.ssh.public_key_openssh +} + +resource "openstack_compute_instance_v2" "elevatevm" { + name = "elevate.github.cpanel.net" + image_id = data.openstack_images_image_ids_v2.images.ids[0] + flavor_name = var.flavor_name + key_pair = openstack_compute_keypair_v2.tf_remote_key.name + user_data = data.template_cloudinit_config.config.rendered + network { + name = "hou-prod-external" + } + + provisioner "remote-exec" { + inline = [<> /root/.ssh/id_ed25519 + echo "${var.ssh_public_key}" >> /root/.ssh/authorized_keys + echo 'waiting on cloud-init...' + cloud-init status --wait > /dev/null || true + EOF + ] + connection { + type = "ssh" + agent = "false" + host = self.access_ip_v4 + user = "root" + script_path = "/root/elevate_bootstrap" + private_key = tls_private_key.ssh.private_key_pem + } + } +} + diff --git a/.github/workflows/openstack/terraform/outputs.tf b/.github/workflows/openstack/terraform/outputs.tf new file mode 100644 index 00000000..396221f0 --- /dev/null +++ b/.github/workflows/openstack/terraform/outputs.tf @@ -0,0 +1,7 @@ +output "address" { + value = openstack_compute_instance_v2.elevatevm.access_ip_v4 +} + +output "id" { + value = openstack_compute_instance_v2.elevatevm.id +} \ No newline at end of file diff --git a/.github/workflows/openstack/terraform/variables.tf b/.github/workflows/openstack/terraform/variables.tf new file mode 100644 index 00000000..e55b66a3 --- /dev/null +++ b/.github/workflows/openstack/terraform/variables.tf @@ -0,0 +1,55 @@ +variable "user" { + type = string + default = "resu" +} + +variable "application_credential_id" { + type = string +} + +variable "application_credential_secret" { + type = string +} + +variable "os_password" { + type = string +} + +variable "os_auth_region" { + type = string +} + +variable "os_auth_url" { + type = string +} + +variable "os_project_domain_name" { + type = string +} + +variable "ssh_private_key" { + type = string + description = "SSH private key matching the public key added to the VMs /root/.ssh/authorized_keys file to allow user access." + sensitive = true +} + +variable "ssh_public_key" { + type = string + description = "SSH public key matching the public key added to the VMs /root/.ssh/authorized_keys file to allow user access." + sensitive = true +} + +variable "image_name" { + type = string + default = "11.110.0.* on CentOS 7" +} + +variable "cpanel_release_version" { + type = string + default = "110" +} + +variable "flavor_name" { + type = string + default = "c2.d20.r2048" +} \ No newline at end of file diff --git a/t/integration/complete.t b/t/integration/complete.t new file mode 100644 index 00000000..0a6f96fb --- /dev/null +++ b/t/integration/complete.t @@ -0,0 +1,23 @@ +#!/usr/local/cpanel/3rdparty/bin/perl + +# Copyright 2024 WebPros International, LLC +# All rights reserved. +# copyright@cpanel.net http://cpanel.net +# This code is subject to the cPanel license. Unauthorized copying is prohibited. + +use lib '/usr/local/cpanel/'; + +use Cpanel::OS (); +use Cpanel::SafeRun::Simple (); + +use Test::More tests => 7; + +use Data::Dumper; + +is( Cpanel::OS->distro(), 'almalinux', 'System is Almalinux after upgrade.' ); +is( Cpanel::OS->major(), '8', 'Verson 8 of OS.' ); +is( -e '/var/log/elevate-cpanel.log', 1, 'Elevate log exists.' ); +like( Cpanel::SafeRun::Simple::saferun( '/bin/sh', '-c', '/scripts/restartsrv_httpd --status' ), qr/is running as root/, 'Apache is up and accepting connections.' ); +like( Cpanel::SafeRun::Simple::saferun( '/bin/sh', '-c', '/scripts/restartsrv_cpsrvd --status' ), qr/is running as root/, 'Chksrvd is up and accepting connections.' ); +like( Cpanel::SafeRun::Simple::saferun( '/bin/sh', '-c', '/scripts/restartsrv_named --status' ), qr/is running as named/, 'Nameserver is up and accepting connections.' ); +ok( Cpanel::SafeRun::Simple::saferun( '/bin/sh', '-c', 'pgrep elevate' ) eq '', 'No instance of elevate-cpanel currently running.' );