Use secret to pull images

DEVELOPMENT.md

@@ -5,6 +5,23 @@ To run agent locally you only need to have correct kubernetes contex and KUBECON
 go run ./cmd/agent
 ```
 
+### Creating image pull secrets
+
+To create secrets for specifying in kvisor config's `pullSecret` attribute there are two options:
+- From credential file:
+```shell
+kubectl -n [namespace] create secret docker-registry [secret-name] \
+  --from-file=.dockerconfigjson=/absolute/path/to/.docker/config.json
+```
+- From credential helper:
+```shell
+kubectl -n [namespace] create secret docker-registry [secret-name] \
+  --docker-server=[registry-server] \
+  --docker-username=[registry-username] \
+  --docker-password=[registry-password]
+```
+Read [more](https://docs.docker.com/engine/reference/commandline/login/#credential-helper-protocol).
+
 ### Running locally on tilt
 
 Start tilt on local kind cluster with mockapi backend which is located in ./tools/mockapi.

Makefile

@@ -1,17 +1,19 @@
+TARGETARCH ?= amd64
+
 build-agent:
-	GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o bin/castai-kvisor ./cmd/agent
+	GOOS=linux GOARCH=$(TARGETARCH) go build -ldflags="-s -w" -o bin/castai-kvisor-$(TARGETARCH) ./cmd/agent
 
 build-agent-github-docker: build-agent
-	docker build -t ghcr.io/castai/kvisor/kvisor:$(IMAGE_TAG) -f Dockerfile.agent .
+	docker buildx build --build-arg TARGETARCH=$(TARGETARCH) --platform linux/$(TARGETARCH) -t ghcr.io/castai/kvisor/kvisor:$(IMAGE_TAG) -f Dockerfile.agent .
 
 push-agent-github-docker: build-agent-github-docker
 	docker push ghcr.io/castai/kvisor/kvisor:$(IMAGE_TAG)
 
 build-imgcollector:
-	GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o bin/castai-imgcollector ./cmd/imgcollector
+	GOOS=linux GOARCH=$(TARGETARCH) go build -ldflags="-s -w" -o bin/castai-imgcollector-$(TARGETARCH) ./cmd/imgcollector
 
 build-imgcollector-github-docker: build-imgcollector
-	docker build -t ghcr.io/castai/kvisor/kvisor-imgcollector:$(IMAGE_TAG) -f Dockerfile.imgcollector .
+	docker buildx build --build-arg TARGETARCH=$(TARGETARCH) --platform linux/$(TARGETARCH) -t ghcr.io/castai/kvisor/kvisor-imgcollector:$(IMAGE_TAG) -f Dockerfile.imgcollector .
 
 push-imgcollector-github-docker: build-imgcollector-github-docker
 	docker push ghcr.io/castai/kvisor/kvisor-imgcollector:$(IMAGE_TAG)

charts/castai-kvisor/values.yaml

@@ -1,5 +1,3 @@
-# Default values for castai-kvisor.
-# This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
 replicas: 1
@@ -25,6 +23,8 @@ image:
 
 imagePullSecrets: {}
 
+imageScanSecret: ""
+
 # Controls `deployment.spec.strategy` field
 updateStrategy:
   type: Recreate
@@ -87,6 +87,9 @@ config: |
     image:
       name: "{{ .Values.image.repository }}-imgcollector:{{ .Values.image.tag | default .Chart.AppVersion }}"
       pullPolicy: IfNotPresent
+    {{ if .Values.imageScanSecret }}
+    pullSecret: "{{ .Values.imageScanSecret }}"
+    {{ end }}
   cloudScan:
     enabled: false
     scanInterval: "1h"

cmd/agent/main.go

@@ -131,19 +131,19 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 		}
 	}()
 
-	restconfig, err := retrieveKubeConfig(logger, cfg.KubeClient.KubeConfigPath)
+	kubeConfig, err := retrieveKubeConfig(logger, cfg.KubeClient.KubeConfigPath)
 	if err != nil {
 		return err
 	}
 
-	restconfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(cfg.KubeClient.QPS), cfg.KubeClient.Burst)
+	kubeConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(cfg.KubeClient.QPS), cfg.KubeClient.Burst)
 
-	clientset, err := kubernetes.NewForConfig(restconfig)
+	clientSet, err := kubernetes.NewForConfig(kubeConfig)
 	if err != nil {
 		return err
 	}
 
-	k8sVersion, err := version.Get(clientset)
+	k8sVersion, err := version.Get(clientSet)
 	if err != nil {
 		return fmt.Errorf("getting kubernetes version: %w", err)
 	}
@@ -228,10 +228,10 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 		if cfg.KubeBench.Force {
 			scannedNodes = []string{}
 		}
-		podLogReader := agentlog.NewPodLogReader(clientset)
+		podLogReader := agentlog.NewPodLogReader(clientSet)
 		objectSubscribers = append(objectSubscribers, kubebench.NewSubscriber(
 			log,
-			clientset,
+			clientSet,
 			cfg.PodNamespace,
 			cfg.Provider,
 			cfg.KubeBench.ScanInterval,
@@ -250,7 +250,7 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 		objectSubscribers = append(objectSubscribers, imagescan.NewSubscriber(
 			log,
 			cfg.ImageScan,
-			imagescan.NewImageScanner(clientset, cfg, deltaState),
+			imagescan.NewImageScanner(clientSet, cfg, deltaState),
 			k8sVersion.MinorInt,
 			deltaState,
 		))
@@ -277,14 +277,14 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 		}
 	}
 
-	gc := jobsgc.NewGC(log, clientset, jobsgc.Config{
+	gc := jobsgc.NewGC(log, clientSet, jobsgc.Config{
 		CleanupInterval: 10 * time.Minute,
 		CleanupJobAge:   10 * time.Minute,
 		Namespace:       cfg.PodNamespace,
 	})
 	go gc.Start(ctx)
 
-	informersFactory := informers.NewSharedInformerFactory(clientset, 0)
+	informersFactory := informers.NewSharedInformerFactory(clientSet, 0)
 	ctrl := controller.New(log, informersFactory, objectSubscribers, k8sVersion)
 
 	telemetryManager := telemetry.NewManager(ctx, log, castaiClient)
@@ -299,7 +299,7 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 	logr := logrusr.New(logger)
 	klog.SetLogger(logr)
 
-	mngr, err := manager.New(restconfig, manager.Options{
+	mngr, err := manager.New(kubeConfig, manager.Options{
 		Logger:                  logr.WithName("manager"),
 		Port:                    cfg.ServicePort,
 		CertDir:                 cfg.CertsDir,

cmd/imgcollector/collector/collector.go

@@ -157,7 +157,31 @@ func (c *Collector) getImage(ctx context.Context) (image.ImageWithIndex, func(),
 	}
 	if c.cfg.Mode == config.ModeRemote {
 		opts := image.DockerOption{}
-		if c.cfg.DockerOptionPath != "" {
+		if c.cfg.ImagePullSecret != "" {
+			configData, err := config.ReadImagePullSecret(os.DirFS(config.SecretMountPath))
+			if err != nil {
+				return nil, nil, fmt.Errorf("reading image pull secret: %w", err)
+			}
+			cfg := image.DockerConfig{}
+			if err := json.Unmarshal(configData, &cfg); err != nil {
+				return nil, nil, fmt.Errorf("parsing image pull secret: %w", err)
+			}
+			if auth, ok := cfg.Auths[imgRef.Context().Registry.Name()]; ok {
+				opts.UserName = auth.Username
+				opts.Password = auth.Password
+				opts.RegistryToken = auth.Token
+			}
+			if auth, ok := cfg.Auths[image.NamespacedRegistry(imgRef)]; ok {
+				opts.UserName = auth.Username
+				opts.Password = auth.Password
+				opts.RegistryToken = auth.Token
+			}
+			if auth, ok := cfg.Auths[fmt.Sprintf("%s/%s", imgRef.Context().RegistryStr(), imgRef.Context().RepositoryStr())]; ok {
+				opts.UserName = auth.Username
+				opts.Password = auth.Password
+				opts.RegistryToken = auth.Token
+			}
+		} else if c.cfg.DockerOptionPath != "" {
 			optsData, err := os.ReadFile(c.cfg.DockerOptionPath)
 			if err != nil {
 				return nil, nil, fmt.Errorf("reading docker options file: %w", err)

cmd/imgcollector/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"io/fs"
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
@@ -26,12 +27,14 @@ const (
 
 const (
 	ContainerdContentDir = "/var/lib/containerd/io.containerd.content.v1.content"
+	SecretMountPath      = "/secret"
 )
 
 type Config struct {
 	ApiURL           string        `envconfig:"KVISOR_SERVER_API_URL" required:"true"`
 	ImageID          string        `envconfig:"COLLECTOR_IMAGE_ID" required:"true"`
 	ImageName        string        `envconfig:"COLLECTOR_IMAGE_NAME" required:"true"`
+	ImagePullSecret  string        `envconfig:"COLLECTOR_PULL_SECRET" default:""`
 	Timeout          time.Duration `envconfig:"COLLECTOR_TIMEOUT" default:"5m"`
 	Mode             Mode          `envconfig:"COLLECTOR_MODE"`
 	Runtime          Runtime       `envconfig:"COLLECTOR_RUNTIME" required:"true"`
@@ -50,3 +53,16 @@ func FromEnv() (Config, error) {
 	}
 	return cfg, nil
 }
+
+// ReadImagePullSecret explicitly mounted at mountPath.
+func ReadImagePullSecret(mount fs.FS) ([]byte, error) {
+	/*
+		apiVersion: v1
+		kind: Secret
+		type: kubernetes.io/dockerconfigjson
+		data:
+			.dockerconfigjson: "<base64 encoded ~/.docker/config.json>"
+	*/
+	// When mounted, data keys become plain text files in the filesystem.
+	return fs.ReadFile(mount, ".dockerconfigjson")
+}

cmd/imgcollector/config/config_test.go

@@ -0,0 +1,26 @@
+package config
+
+import (
+	"encoding/json"
+	"testing"
+	"testing/fstest"
+
+	"github.com/castai/kvisor/cmd/imgcollector/image"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadImagePullSecret(t *testing.T) {
+	r := require.New(t)
+
+	data, err := ReadImagePullSecret(fstest.MapFS{".dockerconfigjson": {
+		Data: []byte(`{"auths": {"ghcr.io": {"username": "username", "password": "password", "auth": "token"}}}`),
+	}})
+	r.NoError(err)
+
+	var cfg image.DockerConfig
+	r.NoError(err, json.Unmarshal(data, &cfg))
+	auth := cfg.Auths["ghcr.io"]
+	r.Equal("username", auth.Username)
+	r.Equal("password", auth.Password)
+	r.Equal("token", auth.Token)
+}

cmd/imgcollector/image/image.go

@@ -2,8 +2,10 @@ package image
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/aquasecurity/trivy/pkg/fanal/types"
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
@@ -34,3 +36,9 @@ func LayerIDs(img v1.Image) ([]string, error) {
 	}
 	return layerIDs, nil
 }
+
+func NamespacedRegistry(ref name.Reference) string {
+	fullName := ref.Context().Name()
+	parts := strings.Split(fullName, "/")
+	return strings.Join(parts[:2], "/")
+}

cmd/imgcollector/image/image_test.go

@@ -0,0 +1,44 @@
+package image
+
+import (
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNamespacedRegistry(t *testing.T) {
+	tests := []struct {
+		name     string
+		ref      name.Reference
+		expected string
+	}{
+		{
+			name:     "Github Container Registry",
+			ref:      name.MustParseReference("ghcr.io/castai/kvisor/kvisor:latest"),
+			expected: "ghcr.io/castai",
+		},
+		{
+			name:     "Gitlab Container Registry",
+			ref:      name.MustParseReference("registry.gitlab.com/castai/kvisor/kvisor:latest"),
+			expected: "registry.gitlab.com/castai",
+		},
+		{
+			name:     "Docker Container Registry Organization",
+			ref:      name.MustParseReference("castai/kvisor"),
+			expected: "index.docker.io/castai",
+		},
+		{
+			name:     "Docker Container Registry",
+			ref:      name.MustParseReference("kvisor"),
+			expected: "index.docker.io/library",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			registry := NamespacedRegistry(test.ref)
+			assert.Equal(t, test.expected, registry)
+		})
+	}
+}

cmd/imgcollector/image/remote.go

@@ -15,6 +15,16 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 )
 
+type DockerConfig struct {
+	Auths map[string]RegistryAuth `json:"auths"`
+}
+
+type RegistryAuth struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+	Token    string `json:"auth"`
+}
+
 type DockerOption struct {
 	// Auth
 	UserName string `yaml:"user_name"`

config/config.go

@@ -80,6 +80,7 @@ type ImageScan struct {
 	Force              bool           `envconfig:"IMAGE_SCAN_FORCE" yaml:"force"`
 	ProfileEnabled     bool           `envconfig:"IMAGE_SCAN_PROFILE_ENABLED" yaml:"profileEnabled"`
 	PhlareEnabled      bool           `envconfig:"IMAGE_SCAN_PHLARE_ENABLED" yaml:"phlareEnabled"`
+	PullSecret         string         `envconfig:"IMAGE_SCAN_PULL_SECRET" yaml:"pullSecret"`
 }
 
 type ImageScanImage struct {

controller/controller.go

@@ -138,7 +138,7 @@ func (c *Controller) transformFunc(i any) (any, error) {
 	obj := i.(Object)
 	// Add missing metadata which is removed by k8s.
 	addObjectMeta(obj)
-	// Remove manged fields since we don't need them. This should decrease memory usage.
+	// Remove managed fields since we don't need them. This should decrease memory usage.
 	obj.SetManagedFields(nil)
 	if _, ok := obj.(*appsv1.DaemonSet); ok {
 		// Remove this fields for ds to fix https://github.com/kubernetes/kubernetes/pull/106388 by custom hashing.

hack/install_agent.sh

@@ -1,20 +1,17 @@
 #!/bin/bash
-
 set -e
 
-local_charts_path="../gh-helm-charts/charts/castai-kvisor"
+default_chart_path="../gh-helm-charts/charts/castai-kvisor"
+chart_path="${CHART_PATH:-$default_chart_path}"
 
 default_api_url="https://api.cast.ai"
 api_url="${API_URL:-$default_api_url}"
 
-default_image_repo="ghcr.io/castai/kvisor"
+default_image_repo="ghcr.io/castai/kvisor/kvisor"
 image_repo="${IMAGE_REPO:-$default_image_repo}"
 
-default_image_tag="alpha1"
-image_tag="${IMAGE_TAG:-$default_image_tag}"
-
 if [ "$API_KEY" == "" ]; then
-  echo "CLUSTER_ID is required"
+  echo "API_KEY is required"
   exit 1
 fi
 
@@ -23,10 +20,22 @@ if [ "$CLUSTER_ID" == "" ]; then
   exit 1
 fi
 
-helm upgrade --install --create-namespace castai-kvisor $local_charts_path --devel \
-	--namespace castai-sec \
+if [ "$SECRET_NAME" == "" ]; then
+  echo "SECRET_NAME is required"
+  exit 1
+fi
+
+if [ "$IMAGE_TAG" == "" ]; then
+  echo "IMAGE_TAG is required"
+  exit 1
+fi
+
+helm upgrade --install --create-namespace castai-kvisor $chart_path --devel \
+	--namespace castai-agent \
 	--set castai.apiURL=${api_url} \
 	--set castai.apiKey=${API_KEY} \
 	--set castai.clusterID=${CLUSTER_ID} \
+	--set 'castai.imagePullSecrets[0].name=castai-kvisor-github' \
 	--set image.repository=${image_repo} \
-	--set image.tag=${image_tag}
+	--set image.tag=${IMAGE_TAG} \
+	--set imageScanSecret=${SECRET_NAME}