feat: limit updates release history (#13)

actions/chart_upsert_handler.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/sirupsen/logrus"
+	"helm.sh/helm/v3/pkg/release"
 	helmdriver "helm.sh/helm/v3/pkg/storage/driver"
 
 	"github.com/castai/cluster-controller/castai"
@@ -34,10 +35,11 @@ func (c *chartUpsertHandler) Handle(ctx context.Context, data interface{}) error
 		return err
 	}
 
-	release, err := c.helm.GetRelease(helm.GetReleaseOptions{
+	rel, err := c.helm.GetRelease(helm.GetReleaseOptions{
 		Namespace:   req.Namespace,
 		ReleaseName: req.ReleaseName,
 	})
+
 	if err != nil {
 		if !errors.Is(err, helmdriver.ErrReleaseNotFound) {
 			return fmt.Errorf("getting helm release %q in namespace %q: %w", req.ReleaseName, req.Namespace, err)
@@ -51,10 +53,22 @@ func (c *chartUpsertHandler) Handle(ctx context.Context, data interface{}) error
 		return err
 	}
 
+	// In case previous update stuck we should rollback it.
+	if rel.Info.Status == release.StatusPendingUpgrade {
+		err = c.helm.Rollback(helm.RollbackOptions{
+			Namespace:   rel.Namespace,
+			ReleaseName: rel.Name,
+		})
+		if err != nil {
+			return err
+		}
+	}
+
 	_, err = c.helm.Upgrade(ctx, helm.UpgradeOptions{
 		ChartSource:     &req.ChartSource,
-		Release:         release,
+		Release:         rel,
 		ValuesOverrides: req.ValuesOverrides,
+		MaxHistory:      3, // Keep last 3 releases history.
 	})
 	return err
 }

actions/chart_upsert_handler_test.go

@@ -48,6 +48,9 @@ func TestChartUpsertHandler(t *testing.T) {
 			Name:      "new-release",
 			Version:   1,
 			Namespace: "test",
+			Info: &release.Info{
+				Status: release.StatusDeployed,
+			},
 		}
 
 		helmMock.EXPECT().GetRelease(helm.GetReleaseOptions{
@@ -59,11 +62,35 @@ func TestChartUpsertHandler(t *testing.T) {
 			ChartSource:     &action.ChartSource,
 			Release:         rel,
 			ValuesOverrides: action.ValuesOverrides,
+			MaxHistory:      3,
 		}).Return(nil, nil)
 
 		r.NoError(handler.Handle(ctx, action))
 	})
 
+	t.Run("rollback previous release before upgrade", func(t *testing.T) {
+		action := chartUpsertAction()
+
+		rel := &release.Release{
+			Name:      "new-release",
+			Version:   1,
+			Namespace: "test",
+			Info: &release.Info{
+				Status: release.StatusPendingUpgrade,
+			},
+		}
+
+		helmMock.EXPECT().GetRelease(gomock.Any()).Return(rel, nil)
+
+		helmMock.EXPECT().Rollback(helm.RollbackOptions{
+			Namespace:   action.Namespace,
+			ReleaseName: action.ReleaseName,
+		}).Return(nil)
+
+		helmMock.EXPECT().Upgrade(ctx, gomock.Any()).Return(nil, nil)
+
+		r.NoError(handler.Handle(ctx, action))
+	})
 }
 
 func chartUpsertAction() *castai.ActionChartUpsert {

hack/remote/deploy.sh

@@ -2,36 +2,23 @@
 
 set -e
 
-if [ -z "$API_KEY" ]; then
-  echo "API_KEY environment variable is not defined"
-  exit 1
-fi
-
-if [ -z "$API_URL" ]; then
-  echo "API_URL environment variable is not defined"
-  exit 1
-fi
-
-if [ -z "$CLUSTER_ID" ]; then
-  echo "CLUSTER_ID environment variable is not defined"
-  exit 1
-fi
-
-# go to git repo root
+# Go to git repo root.
 cd "$(git rev-parse --show-toplevel)"
 
+# Build bo binary and push docker image.
 GOOS=linux go build -o bin/castai-cluster-controller .
-
 DOCKER_IMAGE_REPO=europe-west3-docker.pkg.dev/ci-master-mo3d/tilt/$USER/castai-cluster-controller
 IMAGE_TAG=latest
 docker build -t $DOCKER_IMAGE_REPO:$IMAGE_TAG .
 docker push $DOCKER_IMAGE_REPO:$IMAGE_TAG
 
-helm template cluster-controller castai-helm/castai-cluster-controller \
+# Install local chart and binary.
+LOCAL_CHART_DIR=../gh-helm-charts/charts/castai-cluster-controller
+helm upgrade cluster-controller $LOCAL_CHART_DIR \
   -f ./hack/remote/values.yaml \
   --set image.repository="$DOCKER_IMAGE_REPO" \
-  --set apiKey="$API_KEY" \
-  --set apiURL="$API_URL" \
-  --set clusterID="$CLUSTER_ID" | kubectl apply -f - -n castai-agent
+  --reuse-values \
+  --history-max=3 \
+  -n castai-agent
 
 kubectl rollout restart deployment castai-cluster-controller -n castai-agent

hack/remote/values.yaml

@@ -2,4 +2,4 @@ image:
   pullPolicy: Always
   tag: latest
 resources: null
-createNamespace: true
+createNamespace: false

helm/client.go

@@ -1,4 +1,4 @@
-//go:generate mockgen -destination ./mock/client.go . Client
+//go:generate mockgen -source ./client.go -destination ./mock/client.go . Client
 
 package helm
 
@@ -34,13 +34,19 @@ type UpgradeOptions struct {
 	ChartSource     *castai.ChartSource
 	Release         *release.Release
 	ValuesOverrides map[string]string
+	MaxHistory      int
 }
 
 type GetReleaseOptions struct {
 	Namespace   string
 	ReleaseName string
 }
 
+type RollbackOptions struct {
+	Namespace   string
+	ReleaseName string
+}
+
 func NewClient(log logrus.FieldLogger, loader ChartLoader, restConfig *rest.Config) Client {
 	return &client{
 		log: log,
@@ -57,6 +63,7 @@ func NewClient(log logrus.FieldLogger, loader ChartLoader, restConfig *rest.Conf
 type Client interface {
 	Install(ctx context.Context, opts InstallOptions) (*release.Release, error)
 	Upgrade(ctx context.Context, opts UpgradeOptions) (*release.Release, error)
+	Rollback(opts RollbackOptions) error
 	GetRelease(opts GetReleaseOptions) (*release.Release, error)
 }
 
@@ -124,6 +131,7 @@ func (c *client) Upgrade(ctx context.Context, opts UpgradeOptions) (*release.Rel
 
 	upgrade := action.NewUpgrade(cfg)
 	upgrade.Namespace = namespace
+	upgrade.MaxHistory = opts.MaxHistory
 	name := opts.Release.Name
 
 	// Prepare user value overrides.
@@ -142,6 +150,20 @@ func (c *client) Upgrade(ctx context.Context, opts UpgradeOptions) (*release.Rel
 	return res, nil
 }
 
+func (c *client) Rollback(opts RollbackOptions) error {
+	cfg, err := c.configurationGetter.Get(opts.Namespace)
+	if err != nil {
+		return err
+	}
+
+	rollback := action.NewRollback(cfg)
+	err = rollback.Run(opts.ReleaseName)
+	if err != nil {
+		return fmt.Errorf("chart rollback failed, name=%s, namespace=%s: %w", opts.ReleaseName, opts.Namespace, err)
+	}
+	return nil
+}
+
 func (c *client) GetRelease(opts GetReleaseOptions) (*release.Release, error) {
 	cfg, err := c.configurationGetter.Get(opts.Namespace)
 	if err != nil {