GitHub workflows are stealing jobs from one another. Fix Concurrency …

…settings and jobs. Case RE-750: Aid git workflows in not clobbering jobs from one another. Changelog:
cpanel · Sep 16, 2024 · 90a37b6 · 90a37b6
1 parent cbc95e7
commit 90a37b6
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 45 deletions.
diff --git a/.github/workflows/openstack.yml b/.github/workflows/openstack.yml
@@ -1,3 +1,5 @@
+name: Openstack Test Run
+
 on:
   push:
     branches:
@@ -8,6 +10,10 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   TF_VAR_application_credential_id: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
   TF_VAR_application_credential_secret: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
@@ -159,10 +165,8 @@ jobs:
               chmod -v +x /scripts/elevate-cpanel
               /usr/local/cpanel/cpkeyclt
               /scripts/elevate-cpanel --non-interactive --start &
-              /scripts/elevate-cpanel --log &
-              /scripts/elevate-cpanel --log | awk '/Rebooting into stage 2 of 5/ { print | "exit" }'
 
-    wait_for_stage_2_reboot:
+    wait_for_stage_1_reboot:
       runs-on: self-hosted
       needs: start_elevate
       outputs:
@@ -173,6 +177,38 @@ jobs:
           run: |
             ./ssh_retry ${{ needs.start_elevate.outputs.VM_IP }}
 
+    watch_for_stage_2_reboot:
+      runs-on: self-hosted
+      needs: wait_for_stage_1_reboot
+      outputs:
+        VM_IP: ${{ needs.wait_for_stage_1_reboot.outputs.VM_IP }}
+      steps:
+        - name: Monitor Elevate for Reboot from Stage 1 into Stage 2
+          uses: appleboy/ssh-action@v1.0.3
+          with:
+            host: ${{ needs.wait_for_stage_1_reboot.outputs.VM_IP }}
+            username: 'root'
+            key: ${{ secrets.SSH_PRIVATE_KEY }}
+            port: '22'
+            timeout: 60m
+            command_timeout: 30m
+            debug: true
+            script: |
+              /scripts/status_marker 1
+              /scripts/elevate-cpanel --log &
+              REGEX="/usr/bin/systemctl start elevate-cpanel.service" RETVAL=1 /scripts/reboot_watch
+
+    wait_for_stage_2_reboot:
+      runs-on: self-hosted
+      needs: watch_for_stage_2_reboot
+      outputs:
+        VM_IP: ${{ needs.watch_for_stage_2_reboot.outputs.VM_IP }}
+      steps:
+        - name: Wait For VM to Come Back From Stage 2 Reboot
+          working-directory: "./.github/workflows/openstack/"
+          run: |
+            ./ssh_retry ${{ needs.watch_for_stage_2_reboot.outputs.VM_IP }}
+
     watch_for_stage_3_reboot:
       runs-on: self-hosted
       needs: wait_for_stage_2_reboot
@@ -267,46 +303,16 @@ jobs:
           run: |
             ./ssh_retry ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }}
 
-    watch_for_final_reboot:
+    verify_upgraded_os:
       runs-on: self-hosted
       needs: wait_for_stage_5_reboot
       outputs:
         VM_IP: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }}
-      steps:
-        - name: Watch Elevate for Final Reboot
-          uses: appleboy/ssh-action@v1.0.3
-          with:
-            host: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }}
-            username: 'root'
-            key: ${{ secrets.SSH_PRIVATE_KEY }}
-            port: '22'
-            timeout: 30m
-            command_timeout: 30m
-            script: |
-              /scripts/elevate-cpanel --log &
-              REGEX="Doing final reboot" RETVAL=1 /scripts/reboot_watch
-  
-    wait_for_final_reboot:
-      runs-on: self-hosted
-      needs: watch_for_final_reboot
-      outputs:
-        VM_IP: ${{ needs.watch_for_final_reboot.outputs.VM_IP }}
-      steps:
-        - name: Wait For VM to Come Back From Final Reboot
-          working-directory: "./.github/workflows/openstack/"
-          run: |
-            ./ssh_retry ${{ needs.watch_for_final_reboot.outputs.VM_IP }}
-
-    verify_upgraded_os:
-      runs-on: self-hosted
-      needs: wait_for_final_reboot
-      outputs:
-        VM_IP: ${{ needs.wait_for_final_reboot.outputs.VM_IP }}
       steps:
         - name: Verify End Result Integration Tests
           uses: appleboy/ssh-action@v1.0.3
           with:
-            host: ${{ needs.wait_for_final_reboot.outputs.VM_IP }}
+            host: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP }}
             username: 'root'
             key: ${{ secrets.SSH_PRIVATE_KEY }}
             port: '22'

diff --git a/.github/workflows/openstack/reboot_watch b/.github/workflows/openstack/reboot_watch
@@ -11,20 +11,23 @@ my $RETVAL  = 1;
 my $RETRIES = 0;
 
 # Verify REGEX isn't already in the log before we go into tail mode.  The logodump here is sgnificantly faster than File::Tail.
-
-open LOGFILE, ELEVATE_LOG_PATH;
-my @log = <LOGFILE>;
-close LOGFILE;
-
 # Jump if it's found.
 
-if ( grep { /${REGEX}/m } @log ) {
-    _success_message();
-    exit 0;
+open my $elevate_log_fh, ELEVATE_LOG_PATH;
+
+while ( !eof($elevate_log_fh) ) {
+    if ( /${REGEX}/m =~ readline $elevate_log_fh ) {
+        _success_message();
+        close $elevate_log;
+        exit 0;
+    }
 }
 
+close $elevate_log_fh;
+
 while ( $RETVAL != 0 ) {
     _check_elevate_log_for_regex( ELEVATE_LOG_PATH, ${REGEX}, $RETRIES );
+    exit 0;
 }
 
 sub _check_elevate_log_for_regex {
@@ -34,7 +37,6 @@ sub _check_elevate_log_for_regex {
     while ( defined( $line = $file->read ) ) {
         if ( grep { /$REGEX/m } $line ) {
             _success_message();
-            exit 0;
         }
         $RETRIES++;
     }
@@ -43,5 +45,7 @@ sub _check_elevate_log_for_regex {
 sub _success_message {
     my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime );
     print "## [$time] [INFO]: SUCCESS: Reboot regex ( $REGEX ) found in /var/log/elevate-cpanel.log  ##\n";
-    return 0;
+    exit 0;
 }
+
+exit 0;