From 404ee29510b03810a9343631d7ee0085e1d87d70 Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 11:29:06 +0100
Subject: [PATCH 1/9] feat(ci): add helm upgrade test to the workflow

---
 .github/workflows/tests.yaml | 131 +++++++++++++++++++++++++++++++++--
 1 file changed, 127 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 1c8dbd22..fbf91c62 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -73,8 +73,8 @@ jobs:
           tags: |
             ${{ env.HARBOR_URL }}/${{ env.HARBOR_REPO }}:${{ env.VERSION }}-alpine
 
-  e2e:
-    name: Tests End-to-End on K8s
+  e2e_install:
+    name: Tests End-to-End on K8s (Fresh install)
     needs:
       - build
     runs-on: ubuntu-22.04
@@ -145,9 +145,132 @@ jobs:
           kubectl wait deployment nginx --for condition=Available=True --timeout=30s
           echo "kubectl get cachedimages"
           kubectl get cachedimages
-          if [ $(kubectl get cachedimages -o json | jq ".items[0].status.isCached") ];
+          if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
           then
-              if [ $(kubectl get cachedimages -o json | jq ".items[0].status.usedBy.count") -eq 2 ];
+              if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ];
+              then
+                  echo "Found cached image used by 2 pods"
+              else
+                  echo "Error: pods count should be equal 2"
+                  exit 1
+              fi
+          else
+              echo "Error: image cached status is false"
+              exit 1
+          fi
+          for component in proxy controllers
+          do 
+            echo "Testing $component metrics endpoint"
+            for ip in $(kubectl get po -l "app.kubernetes.io/component=$component" -n kuik-system -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
+            do
+               attempts=0
+               success=false
+               while [[ $attempts -lt 3 && $success == false ]]
+               do
+                   response=$(kubectl run curl-pod --image=curlimages/curl --rm -ti --quiet --restart=Never -- curl -s -o /dev/null -w "%{http_code}\n" http://$ip:8080/metrics)
+                   if [[ -z "$response" ]]; then
+                        echo "No HTTP response received from $ip"
+                   elif [[ $response -ge 200 && $response -lt 300 ]]; then
+                        echo "HTTP status code $response is valid for $ip"
+                        success=true
+                   else
+                        echo "HTTP status code $response is not valid for $ip"
+                   fi
+                   attempts=$(( $attempts + 1 ))
+                   sleep 3
+               done
+               if [[ $success == false ]]; then
+                   echo "Failed after 3 attempts for $ip"
+                   exit 1
+               fi
+            done
+          done
+
+  e2e_upgrade:
+    name: Tests End-to-End on K8s (Upgrade)
+    needs:
+      - build
+      - e2e_install
+    runs-on: ubuntu-22.04
+    env:
+      VERSION: ${{ github.run_id }}
+      HARBOR_IMAGE: "harbor.enix.io/kube-image-keeper/kube-image-keeper"
+      HARBOR_REGISTRY: "harbor.enix.io"
+      HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
+      HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
+    strategy:
+      max-parallel: 6
+      matrix:
+        k8sversion: ["v1.24.15", "v1.25.11", "v1.26.6", "v1.27.3", "v1.28.0", "v1.29.0"]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup KinD
+        uses: helm/kind-action@v1.9.0
+        with:
+          node_image: kindest/node:${{ matrix.k8sversion }}
+
+      - name: Run cert-manager installation
+        run: |
+          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
+          kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=30s
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.6.1
+
+      - name: Set up helm
+        uses: azure/setup-helm@v4
+        with:
+          version: '3.9.0'
+
+      - name: Run chart-testing (lint)
+        run: |
+          set -euo pipefail
+          ct lint \
+            --charts helm/kube-image-keeper \
+            --chart-repos bitnami=https://charts.bitnami.com/bitnami \
+            --validate-maintainers=false --check-version-increment=false
+
+# Need wait for the next release with flash --skip-clean-up
+#      - name: Run chart-testing (install)
+#        run: |
+#          set -euo pipefail
+#          ct install \
+#            --charts helm/cache-registry \
+#            --helm-extra-set-args "--set controllers.image.tag=latest --set proxy.image.tag=latest"
+
+
+      - name: Run helm (install latest release)
+        run : |
+          set -euo pipefail
+          helm repo add enix https://charts.enix.io/
+          helm repo update
+          helm upgrade --install kube-image-keeper -n kuik-system --create-namespace enix/kube-image-keeper --debug
+          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=30s
+          kubectl get po -n kuik-system
+
+      - name: Run helm (upgrade)
+        run : |
+          set -euo pipefail
+          kubectl create secret docker-registry harbor-secret -n kuik-system --docker-server=${{ env.HARBOR_REGISTRY }} \
+            --docker-username="$HARBOR_USERNAME" --docker-password="$HARBOR_PASSWORD"
+          helm upgrade --install kube-image-keeper -n kuik-system --create-namespace ./helm/kube-image-keeper \
+            --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \
+            --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \
+            --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret  --debug
+          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=30s
+
+      - name: Run end-to-end tests
+        run: |
+          set -euo pipefail
+          kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
+          kubectl wait deployment nginx --for condition=Available=True --timeout=30s
+          echo "kubectl get cachedimages"
+          kubectl get cachedimages
+          if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
+          then
+              if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ];
               then
                   echo "Found cached image used by 2 pods"
               else

From 4ca72112df6ca81dd9ff34f3c328723112ec7a12 Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 16:37:38 +0100
Subject: [PATCH 2/9] ci: wip upgrade tests

---
 .github/workflows/tests.yaml | 47 ++++++++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index fbf91c62..3d5128cc 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -74,7 +74,7 @@ jobs:
             ${{ env.HARBOR_URL }}/${{ env.HARBOR_REPO }}:${{ env.VERSION }}-alpine
 
   e2e_install:
-    name: Tests End-to-End on K8s (Fresh install)
+    name: Tests e2e on K8s (Fresh install)
     needs:
       - build
     runs-on: ubuntu-22.04
@@ -100,7 +100,7 @@ jobs:
       - name: Run cert-manager installation
         run: |
           kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
-          kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=30s
+          kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=90s
 
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.6.1
@@ -136,15 +136,20 @@ jobs:
             --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \
             --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \
             --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret  --debug
-          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=30s
+          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=90s
+          helm history kube-image-keeper -n kuik-system
 
       - name: Run end-to-end tests
         run: |
           set -euo pipefail
           kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
+          kubectl rollout status deploy nginx 
           kubectl wait deployment nginx --for condition=Available=True --timeout=30s
-          echo "kubectl get cachedimages"
+          echo "kubectl get cachedimage"
           kubectl get cachedimages
+          echo "kubectl get repository"
+          kubectl get repository
+          ## Check if our test image is cached
           if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
           then
               if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ];
@@ -158,6 +163,15 @@ jobs:
               echo "Error: image cached status is false"
               exit 1
           fi
+          ## Check repository status
+          if kubectl get repository docker.io-library-nginx -o json | jq '.status.phase == "Ready"' ;
+          then
+              echo "Found repository"
+          else
+              echo "Error: image repository status is not Ready"
+              exit 1
+          fi
+          ## Check for kuik's components metrics
           for component in proxy controllers
           do 
             echo "Testing $component metrics endpoint"
@@ -187,7 +201,7 @@ jobs:
           done
 
   e2e_upgrade:
-    name: Tests End-to-End on K8s (Upgrade)
+    name: Tests e2e on K8s (Upgrade)
     needs:
       - build
       - e2e_install
@@ -258,16 +272,24 @@ jobs:
           helm upgrade --install kube-image-keeper -n kuik-system --create-namespace ./helm/kube-image-keeper \
             --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \
             --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \
-            --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret  --debug
-          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=30s
+            --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret --wait --debug
+          kubectl rollout status deploy kube-image-keeper-controllers -n kuik-system
+          kubectl rollout status ds kube-image-keeper-proxy -n kuik-system
+          kubectl get po -n kuik-system
+          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=90s
+          helm history kube-image-keeper -n kuik-system
 
       - name: Run end-to-end tests
         run: |
           set -euo pipefail
           kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
+          kubectl rollout status deploy nginx 
           kubectl wait deployment nginx --for condition=Available=True --timeout=30s
-          echo "kubectl get cachedimages"
+          echo "kubectl get cachedimage"
           kubectl get cachedimages
+          echo "kubectl get repository"
+          kubectl get repository
+          ## Check if our test image is cached
           if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
           then
               if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ];
@@ -281,6 +303,15 @@ jobs:
               echo "Error: image cached status is false"
               exit 1
           fi
+          ## Check repository status
+          if kubectl get repository docker.io-library-nginx -o json | jq '.status.phase == "NotReady"' ;
+          then
+              echo "Found repository"
+          else
+              echo "Error: image repository status is not Ready"
+              exit 1
+          fi
+          ## Check for kuik's components metrics
           for component in proxy controllers
           do 
             echo "Testing $component metrics endpoint"

From 6ca5997d894f52f549549a1de3dba56cb11e576a Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 16:55:59 +0100
Subject: [PATCH 3/9] fix(docs): update docker registry documentation link

---
 docs/high-availability.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/high-availability.md b/docs/high-availability.md
index 7cb329b2..fccf0a01 100644
--- a/docs/high-availability.md
+++ b/docs/high-availability.md
@@ -50,7 +50,7 @@ registry:
       bucket: registry
 ```
 
-Please refer to the [Docker registry S3 documentation](https://github.com/docker/docs/blob/main/registry/storage-drivers/s3.md) for more details.
+Please refer to the [Docker registry S3 documentation](https://github.com/distribution/distribution/blob/main/docs/content/storage-drivers/s3.md) for more details.
 
 Note that when using AWS S3 buckets, you shouldn't prefix the bucket name with `s3://`:
 

From e564d4e626469269a9a835b700e21d4679105381 Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 17:17:04 +0100
Subject: [PATCH 4/9] refactor(ci): tests

---
 .github/workflows/tests.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 3d5128cc..ded8a5a2 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -164,7 +164,7 @@ jobs:
               exit 1
           fi
           ## Check repository status
-          if kubectl get repository docker.io-library-nginx -o json | jq '.status.phase == "Ready"' ;
+          if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ;
           then
               echo "Found repository"
           else
@@ -275,8 +275,6 @@ jobs:
             --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret --wait --debug
           kubectl rollout status deploy kube-image-keeper-controllers -n kuik-system
           kubectl rollout status ds kube-image-keeper-proxy -n kuik-system
-          kubectl get po -n kuik-system
-          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=90s
           helm history kube-image-keeper -n kuik-system
 
       - name: Run end-to-end tests
@@ -304,7 +302,7 @@ jobs:
               exit 1
           fi
           ## Check repository status
-          if kubectl get repository docker.io-library-nginx -o json | jq '.status.phase == "NotReady"' ;
+          if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"NotReady"' ] ;
           then
               echo "Found repository"
           else

From 1b7022c88ebcdb31182c49e58f85895caf0b0fb5 Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 17:25:33 +0100
Subject: [PATCH 5/9] feat(Dockerfile): update upstream image version

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 31f5ec1c..b77c3c45 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=${BUILDPLATFORM} golang:1.20-alpine3.17 AS builder
+FROM --platform=${BUILDPLATFORM} golang:1.20-alpine3.19 AS builder
 
 WORKDIR /workspace
 
@@ -40,7 +40,7 @@ RUN --mount=type=cache,target="/root/.cache/go-build" \
     go build -ldflags="$LD_FLAGS" -o manager cmd/cache/main.go && \
     go build -ldflags="$LD_FLAGS" -o registry-proxy cmd/proxy/main.go
 
-FROM alpine:3.17 AS alpine
+FROM alpine:3.19 AS alpine
 
 COPY --from=builder /workspace/manager /usr/local/bin/
 COPY --from=builder /workspace/registry-proxy /usr/local/bin/

From 0c289f791d5e67e53fb11af5a88431271f960c94 Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 17:28:49 +0100
Subject: [PATCH 6/9] feat(helm): use latest stable docker registry version

---
 .github/workflows/tests.yaml       | 2 +-
 helm/kube-image-keeper/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index ded8a5a2..caa165a8 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -302,7 +302,7 @@ jobs:
               exit 1
           fi
           ## Check repository status
-          if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"NotReady"' ] ;
+          if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ;
           then
               echo "Found repository"
           else
diff --git a/helm/kube-image-keeper/values.yaml b/helm/kube-image-keeper/values.yaml
index b6365eaa..0603686e 100644
--- a/helm/kube-image-keeper/values.yaml
+++ b/helm/kube-image-keeper/values.yaml
@@ -198,7 +198,7 @@ registry:
     # -- Registry image pull policy
     pullPolicy: IfNotPresent
     # -- Registry image tag
-    tag: "2.8.2"
+    tag: "2.8"
   # -- Number of replicas for the registry pod
   replicas: 1
   persistence:

From 4861c15159858d7c9942e5e5010277404efd3d4c Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 17:34:25 +0100
Subject: [PATCH 7/9] ci: split tests by step

---
 .github/workflows/tests.yaml | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index caa165a8..6a8bebfb 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -139,7 +139,7 @@ jobs:
           kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=90s
           helm history kube-image-keeper -n kuik-system
 
-      - name: Run end-to-end tests
+      - name: Deploy test container
         run: |
           set -euo pipefail
           kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
@@ -149,6 +149,10 @@ jobs:
           kubectl get cachedimages
           echo "kubectl get repository"
           kubectl get repository
+
+      - name: Test cachedimage (CRD)
+        run: |
+          set -euo pipefail
           ## Check if our test image is cached
           if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
           then
@@ -163,6 +167,10 @@ jobs:
               echo "Error: image cached status is false"
               exit 1
           fi
+
+      - name: Test repository (CRD)
+        run: |
+          set -euo pipefail
           ## Check repository status
           if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ;
           then
@@ -171,6 +179,10 @@ jobs:
               echo "Error: image repository status is not Ready"
               exit 1
           fi
+
+      - name: Test metrics endpoint
+        run: |
+          set -euo pipefail
           ## Check for kuik's components metrics
           for component in proxy controllers
           do 
@@ -277,7 +289,7 @@ jobs:
           kubectl rollout status ds kube-image-keeper-proxy -n kuik-system
           helm history kube-image-keeper -n kuik-system
 
-      - name: Run end-to-end tests
+      - name: Deploy test container
         run: |
           set -euo pipefail
           kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
@@ -287,6 +299,10 @@ jobs:
           kubectl get cachedimages
           echo "kubectl get repository"
           kubectl get repository
+
+      - name: Test cachedimage (CRD)
+        run: |
+          set -euo pipefail
           ## Check if our test image is cached
           if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
           then
@@ -301,6 +317,10 @@ jobs:
               echo "Error: image cached status is false"
               exit 1
           fi
+
+      - name: Test repository (CRD)
+        run: |
+          set -euo pipefail
           ## Check repository status
           if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ;
           then
@@ -309,6 +329,10 @@ jobs:
               echo "Error: image repository status is not Ready"
               exit 1
           fi
+
+      - name: Test metrics endpoint
+        run: |
+          set -euo pipefail
           ## Check for kuik's components metrics
           for component in proxy controllers
           do 
@@ -337,3 +361,4 @@ jobs:
                fi
             done
           done
+

From 7ae6e03ccad7dcec9e2254915e0627675a8f471e Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Tue, 19 Mar 2024 17:41:30 +0100
Subject: [PATCH 8/9] feat(ci): update Kind image version

---
 .github/workflows/tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 6a8bebfb..bd493e18 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -87,7 +87,7 @@ jobs:
     strategy:
       max-parallel: 6
       matrix:
-        k8sversion: ["v1.24.15", "v1.25.11", "v1.26.6", "v1.27.3", "v1.28.0", "v1.29.0"]
+        k8sversion: ["v1.24.17", "v1.25.16", "v1.26.14", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -227,7 +227,7 @@ jobs:
     strategy:
       max-parallel: 6
       matrix:
-        k8sversion: ["v1.24.15", "v1.25.11", "v1.26.6", "v1.27.3", "v1.28.0", "v1.29.0"]
+        k8sversion: ["v1.24.17", "v1.25.16", "v1.26.14", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4

From c5238cf36db18aab7c45a158c1462967c382f41d Mon Sep 17 00:00:00 2001
From: David Donchez <david.donchez@enix.fr>
Date: Wed, 20 Mar 2024 10:05:24 +0100
Subject: [PATCH 9/9] feat(ci): backport our e2e tests to the release workflow

---
 .github/workflows/release.yml | 221 +++++++++++++++++++++++++++++++---
 1 file changed, 204 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6902bb54..aa9b7a48 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -173,8 +173,9 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           tags: |
             ${{ env.HARBOR_URL }}/${{ env.HARBOR_REPO }}:${{ env.VERSION }}-alpine
-  e2e:
-    name: Tests End-to-End on K8s
+
+  e2e_install:
+    name: Tests e2e on K8s (Fresh install)
     needs:
       - build
     runs-on: ubuntu-22.04
@@ -187,7 +188,7 @@ jobs:
     strategy:
       max-parallel: 6
       matrix:
-        k8sversion: ["v1.24.15", "v1.25.11", "v1.26.6", "v1.27.3", "v1.28.0", "v1.29.0"]
+        k8sversion: ["v1.24.17", "v1.25.16", "v1.26.14", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -200,7 +201,7 @@ jobs:
       - name: Run cert-manager installation
         run: |
           kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
-          kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=30s
+          kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=90s
 
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.6.1
@@ -218,14 +219,6 @@ jobs:
             --chart-repos bitnami=https://charts.bitnami.com/bitnami \
             --validate-maintainers=false --check-version-increment=false
 
-# Need wait for the next release with flag --skip-clean-up
-#      - name: Run chart-testing (install)
-#        run: |
-#          set -euo pipefail
-#          ct install \
-#            --charts helm/kube-image-keeper \
-#            --helm-extra-set-args "--set controllers.image.tag=latest --set proxy.image.tag=latest"
-
       - name: Run helm (install)
         run : |
           set -euo pipefail
@@ -236,18 +229,168 @@ jobs:
             --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \
             --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \
             --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret  --debug
+          kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=90s
+          helm history kube-image-keeper -n kuik-system
+
+      - name: Deploy test container
+        run: |
+          set -euo pipefail
+          kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
+          kubectl rollout status deploy nginx
+          kubectl wait deployment nginx --for condition=Available=True --timeout=30s
+          echo "kubectl get cachedimage"
+          kubectl get cachedimages
+          echo "kubectl get repository"
+          kubectl get repository
+
+      - name: Test cachedimage (CRD)
+        run: |
+          set -euo pipefail
+          ## Check if our test image is cached
+          if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
+          then
+              if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ];
+              then
+                  echo "Found cached image used by 2 pods"
+              else
+                  echo "Error: pods count should be equal 2"
+                  exit 1
+              fi
+          else
+              echo "Error: image cached status is false"
+              exit 1
+          fi
+
+      - name: Test repository (CRD)
+        run: |
+          set -euo pipefail
+          ## Check repository status
+          if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ;
+          then
+              echo "Found repository"
+          else
+              echo "Error: image repository status is not Ready"
+              exit 1
+          fi
+
+      - name: Test metrics endpoint
+        run: |
+          set -euo pipefail
+          ## Check for kuik's components metrics
+          for component in proxy controllers
+          do
+            echo "Testing $component metrics endpoint"
+            for ip in $(kubectl get po -l "app.kubernetes.io/component=$component" -n kuik-system -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
+            do
+               attempts=0
+               success=false
+               while [[ $attempts -lt 3 && $success == false ]]
+               do
+                   response=$(kubectl run curl-pod --image=curlimages/curl --rm -ti --quiet --restart=Never -- curl -s -o /dev/null -w "%{http_code}\n" http://$ip:8080/metrics)
+                   if [[ -z "$response" ]]; then
+                        echo "No HTTP response received from $ip"
+                   elif [[ $response -ge 200 && $response -lt 300 ]]; then
+                        echo "HTTP status code $response is valid for $ip"
+                        success=true
+                   else
+                        echo "HTTP status code $response is not valid for $ip"
+                   fi
+                   attempts=$(( $attempts + 1 ))
+                   sleep 3
+               done
+               if [[ $success == false ]]; then
+                   echo "Failed after 3 attempts for $ip"
+                   exit 1
+               fi
+            done
+          done
+
+  e2e_upgrade:
+    name: Tests e2e on K8s (Upgrade)
+    needs:
+      - build
+      - e2e_install
+    runs-on: ubuntu-22.04
+    env:
+      VERSION: ${{ github.run_id }}
+      HARBOR_IMAGE: "harbor.enix.io/kube-image-keeper/kube-image-keeper"
+      HARBOR_REGISTRY: "harbor.enix.io"
+      HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
+      HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
+    strategy:
+      max-parallel: 6
+      matrix:
+        k8sversion: ["v1.24.17", "v1.25.16", "v1.26.14", "v1.27.11", "v1.28.7", "v1.29.2"]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup KinD
+        uses: helm/kind-action@v1.9.0
+        with:
+          node_image: kindest/node:${{ matrix.k8sversion }}
+
+      - name: Run cert-manager installation
+        run: |
+          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
+          kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=30s
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.6.1
+
+      - name: Set up helm
+        uses: azure/setup-helm@v4
+        with:
+          version: '3.9.0'
+
+      - name: Run chart-testing (lint)
+        run: |
+          set -euo pipefail
+          ct lint \
+            --charts helm/kube-image-keeper \
+            --chart-repos bitnami=https://charts.bitnami.com/bitnami \
+            --validate-maintainers=false --check-version-increment=false
+
+      - name: Run helm (install latest release)
+        run : |
+          set -euo pipefail
+          helm repo add enix https://charts.enix.io/
+          helm repo update
+          helm upgrade --install kube-image-keeper -n kuik-system --create-namespace enix/kube-image-keeper --debug
           kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=30s
+          kubectl get po -n kuik-system
+
+      - name: Run helm (upgrade)
+        run : |
+          set -euo pipefail
+          kubectl create secret docker-registry harbor-secret -n kuik-system --docker-server=${{ env.HARBOR_REGISTRY }} \
+            --docker-username="$HARBOR_USERNAME" --docker-password="$HARBOR_PASSWORD"
+          helm upgrade --install kube-image-keeper -n kuik-system --create-namespace ./helm/kube-image-keeper \
+            --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \
+            --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \
+            --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret --wait --debug
+          kubectl rollout status deploy kube-image-keeper-controllers -n kuik-system
+          kubectl rollout status ds kube-image-keeper-proxy -n kuik-system
+          helm history kube-image-keeper -n kuik-system
 
-      - name: Run end-to-end tests
+      - name: Deploy test container
         run: |
           set -euo pipefail
           kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2
+          kubectl rollout status deploy nginx
           kubectl wait deployment nginx --for condition=Available=True --timeout=30s
-          echo "kubectl get cachedimages"
+          echo "kubectl get cachedimage"
           kubectl get cachedimages
-          if [ $(kubectl get cachedimages -o json | jq ".items[0].status.isCached") ];
+          echo "kubectl get repository"
+          kubectl get repository
+
+      - name: Test cachedimage (CRD)
+        run: |
+          set -euo pipefail
+          ## Check if our test image is cached
+          if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ];
           then
-              if [ $(kubectl get cachedimages -o json | jq ".items[0].status.usedBy.count") -eq 2 ];
+              if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ];
               then
                   echo "Found cached image used by 2 pods"
               else
@@ -259,10 +402,54 @@ jobs:
               exit 1
           fi
 
+      - name: Test repository (CRD)
+        run: |
+          set -euo pipefail
+          ## Check repository status
+          if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ;
+          then
+              echo "Found repository"
+          else
+              echo "Error: image repository status is not Ready"
+              exit 1
+          fi
+
+      - name: Test metrics endpoint
+        run: |
+          set -euo pipefail
+          ## Check for kuik's components metrics
+          for component in proxy controllers
+          do
+            echo "Testing $component metrics endpoint"
+            for ip in $(kubectl get po -l "app.kubernetes.io/component=$component" -n kuik-system -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
+            do
+               attempts=0
+               success=false
+               while [[ $attempts -lt 3 && $success == false ]]
+               do
+                   response=$(kubectl run curl-pod --image=curlimages/curl --rm -ti --quiet --restart=Never -- curl -s -o /dev/null -w "%{http_code}\n" http://$ip:8080/metrics)
+                   if [[ -z "$response" ]]; then
+                        echo "No HTTP response received from $ip"
+                   elif [[ $response -ge 200 && $response -lt 300 ]]; then
+                        echo "HTTP status code $response is valid for $ip"
+                        success=true
+                   else
+                        echo "HTTP status code $response is not valid for $ip"
+                   fi
+                   attempts=$(( $attempts + 1 ))
+                   sleep 3
+               done
+               if [[ $success == false ]]; then
+                   echo "Failed after 3 attempts for $ip"
+                   exit 1
+               fi
+            done
+          done
+
   release:
     name: Release
     needs:
-      - e2e
+      - e2e_upgrade
       - semver
     runs-on: ubuntu-22.04
     outputs: