Skip to content

Commit

Permalink
Include managed serviceaccount (#133)
Browse files Browse the repository at this point in the history
* Start managed serviceaccount embedded

Signed-off-by: clyang82 <chuyang@redhat.com>

* Add e2e tests

Signed-off-by: clyang82 <chuyang@redhat.com>

* fix make check

Signed-off-by: clyang82 <chuyang@redhat.com>

---------

Signed-off-by: clyang82 <chuyang@redhat.com>
  • Loading branch information
clyang82 authored Mar 13, 2024
1 parent 03cc671 commit 1001b0e
Show file tree
Hide file tree
Showing 18 changed files with 530 additions and 28 deletions.
1 change: 0 additions & 1 deletion cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"os"

"github.com/spf13/cobra"

utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/cli"
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ require (
k8s.io/metrics v0.29.2
k8s.io/utils v0.0.0-20240102154912-e7106e64919e
open-cluster-management.io/api v0.13.0
open-cluster-management.io/managed-serviceaccount v0.5.0
open-cluster-management.io/ocm v0.13.0
sigs.k8s.io/controller-runtime v0.17.2
)
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,8 @@ open-cluster-management.io/addon-framework v0.8.1-0.20240205013730-13fbb6259464
open-cluster-management.io/addon-framework v0.8.1-0.20240205013730-13fbb6259464/go.mod h1:SBs6wF0Umzr5/miJb9p8uMaTDbcjphHHQLa76nXnbU8=
open-cluster-management.io/api v0.13.0 h1:dlcJEZlNlE0DmSDctK2s7iWKg9l+Tgb0V78Z040nMuk=
open-cluster-management.io/api v0.13.0/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
open-cluster-management.io/managed-serviceaccount v0.5.0 h1:yEFTlwPp3tecorzHLwa71mIJzwQtxcwRUnUhRF4OX0U=
open-cluster-management.io/managed-serviceaccount v0.5.0/go.mod h1:pvSKkwFynokhtV7ksN1z0BNWQ37bG8FudOfmjn55ciA=
open-cluster-management.io/ocm v0.13.0 h1:VIaHhJrxChppH3boJ2/AS0/OXjEZaeFDnywaUrzSjV8=
open-cluster-management.io/ocm v0.13.0/go.mod h1:rZZY6F3r1f3/91Z/XOZErICLjOX7T3G2hfN7NvIYNaw=
open-cluster-management.io/sdk-go v0.13.0 h1:ddMGsPUekQr9z03tVN6vF39Uf+WEKMtGU/xSd81HdoA=
Expand Down
2 changes: 2 additions & 0 deletions hack/deploy-multicluster-controlplane.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ image=${IMAGE_NAME:-"quay.io/stolostron/multicluster-controlplane:latest"}
external_hostname=${EXTERNAL_HOSTNAME:-""}
node_port=${NODE_PORT:-0}
etcd_mod=${ETCD_MOD:-""}
feature_gates=${FEATURE_GATES:-"DefaultClusterSet=true,ManagedClusterAutoApproval=true"}

if [ "$uninstall"x = "uninstall"x ]; then
helm -n ${HUB_NAME} uninstall multicluster-controlplane
Expand All @@ -33,6 +34,7 @@ args="--create-namespace"
args="$args --set enableSelfManagement=${self_management}"
args="$args --set image=${image}"
args="$args --set autoApprovalBootstrapUsers=system:admin"
args="$args --set features=${feature_gates}"

if [ 0 -eq $node_port ]; then
args="$args --set route.enabled=true"
Expand Down
6 changes: 6 additions & 0 deletions hack/deploy/agent/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,9 @@ rules:
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["impersonate"]
- apiGroups: [""]
resources: ["serviceaccounts", "serviceaccounts/token"]
verbs: ["get", "watch", "list", "create", "delete"]
- apiGroups: ["authentication.k8s.io"]
resources: ["tokenreviews"]
verbs: ["create"]
1 change: 1 addition & 0 deletions hack/deploy/agent/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ spec:
- "agent"
- "--cluster-name=loopback"
- "--bootstrap-kubeconfig=/spoke/bootstrap/kubeconfig"
- "--feature-gates=ManagedServiceAccount=true"
securityContext:
allowPrivilegeEscalation: false
capabilities:
Expand Down
90 changes: 90 additions & 0 deletions pkg/agent/addons/managedserviceaccount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright Contributors to the Open Cluster Management project

package addons

import (
"context"
"fmt"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"

"open-cluster-management.io/managed-serviceaccount/pkg/addon/agent/controller"
"open-cluster-management.io/managed-serviceaccount/pkg/common"
"open-cluster-management.io/multicluster-controlplane/pkg/util"
)

func StartManagedServiceAccountAgent(ctx context.Context, hubMgr manager.Manager, clusterName string) error {
spokeNamespace := util.GetComponentNamespace()

hubNativeClient, err := kubernetes.NewForConfig(hubMgr.GetConfig())
if err != nil {
return fmt.Errorf("unable to instantiate a kubernetes native client")
}

spokeCfg, err := rest.InClusterConfig()
if err != nil {
return fmt.Errorf("failed build a in-cluster spoke cluster client config")
}

spokeNativeClient, err := kubernetes.NewForConfig(spokeCfg)
if err != nil {
return fmt.Errorf("unable to build a spoke kubernetes client")
}

resources, err := spokeNativeClient.Discovery().ServerResourcesForGroupVersion("v1")
if err != nil {
return fmt.Errorf("failed api discovery in the spoke cluster: %v", err)
}
found := false
for _, r := range resources.APIResources {
if r.Kind == "TokenRequest" {
found = true
}
}
if !found {
return fmt.Errorf(`no "serviceaccounts/token" resource discovered in the managed cluster,` +
`is --service-account-signing-key-file configured for the kube-apiserver?`)
}

spokeCache, err := cache.New(spokeCfg, cache.Options{
ByObject: map[client.Object]cache.ByObject{
&corev1.ServiceAccount{}: {
Namespaces: map[string]cache.Config{
spokeNamespace: {
LabelSelector: labels.SelectorFromSet(
labels.Set{
common.LabelKeyIsManagedServiceAccount: "true",
},
),
},
},
},
},
})
if err != nil {
return fmt.Errorf("unable to instantiate a spoke serviceaccount cache")
}
if err = hubMgr.Add(spokeCache); err != nil {
return fmt.Errorf("unable to add spoke cache to manager")
}

ctrl := controller.TokenReconciler{
ClusterName: clusterName,
Cache: hubMgr.GetCache(),
HubClient: hubMgr.GetClient(),
HubNativeClient: hubNativeClient,
SpokeNamespace: spokeNamespace,
SpokeNativeClient: spokeNativeClient,
SpokeClientConfig: spokeCfg,
SpokeCache: spokeCache,
}

return ctrl.SetupWithManager(hubMgr)
}
79 changes: 72 additions & 7 deletions pkg/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,36 @@ import (
"embed"
"fmt"

"github.com/openshift/library-go/pkg/assets"
"github.com/openshift/library-go/pkg/controller/controllercmd"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
"github.com/spf13/pflag"

crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/informers"
kubescheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"

"github.com/openshift/library-go/pkg/assets"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"

clusterv1informers "open-cluster-management.io/api/client/cluster/informers/externalversions"
"open-cluster-management.io/multicluster-controlplane/pkg/util"
authv1beta1 "open-cluster-management.io/managed-serviceaccount/apis/authentication/v1beta1"
commonoptions "open-cluster-management.io/ocm/pkg/common/options"
"open-cluster-management.io/ocm/pkg/features"
registrationspoke "open-cluster-management.io/ocm/pkg/registration/spoke"
singletonspoke "open-cluster-management.io/ocm/pkg/singleton/spoke"
workspoke "open-cluster-management.io/ocm/pkg/work/spoke"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/manager"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"

"open-cluster-management.io/multicluster-controlplane/pkg/agent/addons"
mcfeature "open-cluster-management.io/multicluster-controlplane/pkg/feature"
"open-cluster-management.io/multicluster-controlplane/pkg/util"
)

//go:embed crds
Expand All @@ -48,6 +54,8 @@ var (

func init() {
utilruntime.Must(crdv1.AddToScheme(genericScheme))
utilruntime.Must(kubescheme.AddToScheme(genericScheme))
utilruntime.Must(authv1beta1.AddToScheme(genericScheme))
}

type AgentOptions struct {
Expand Down Expand Up @@ -182,3 +190,60 @@ func (o *AgentOptions) ensureCRDs(ctx context.Context, client apiextensionsclien

return nil
}

// RunAddOns runs the addons in the agent
func (a *AgentOptions) RunAddOns(ctx context.Context) error {

startCtrlMgr := false

clusterName := a.CommonOpts.SpokeClusterName

// TODO should use hubKubeConfig, err := clientcmd.BuildConfigFromFlags("", a.WorkAgentOpts.WorkloadSourceDriver.Config)
hubKubeConfig, err := clientcmd.BuildConfigFromFlags("", a.RegistrationAgentOpts.BootstrapKubeconfig)
if err != nil {
return fmt.Errorf("unable to load kubeconfig from file %q: %v", a.KubeConfig, err)
}

hubManager, err := a.newHubManager(hubKubeConfig)
if err != nil {
return err
}

if features.SpokeMutableFeatureGate.Enabled(mcfeature.ManagedServiceAccount) {
klog.Info("starting managed serviceaccount addon agent")
if err := addons.StartManagedServiceAccountAgent(ctx, hubManager, clusterName); err != nil {
klog.Fatalf("failed to setup managed serviceaccount addon, %v", err)
}

startCtrlMgr = true
}

if !startCtrlMgr {
return nil
}

go func() {
klog.Info("starting the embedded hub controller-runtime manager in controlplane agent")
if err := hubManager.Start(ctx); err != nil {
klog.Fatalf("failed to start embedded hub controller-runtime manager, %v", err)
}
<-ctx.Done()
}()

return nil
}

func (a *AgentOptions) newHubManager(hubKubeConfig *rest.Config) (manager.Manager, error) {
mgr, err := ctrl.NewManager(hubKubeConfig, ctrl.Options{
Scheme: genericScheme,
Metrics: metricsserver.Options{
BindAddress: "0", //TODO think about the mertics later
},
Logger: ctrl.Log.WithName("ctrl-runtime-manager"),
})
if err != nil {
return nil, err
}

return mgr, nil
}
18 changes: 13 additions & 5 deletions pkg/cmd/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,21 @@ package agent
import (
"context"

utilruntime "k8s.io/apimachinery/pkg/util/runtime"
ocmfeature "open-cluster-management.io/api/feature"
"open-cluster-management.io/ocm/pkg/features"

"github.com/spf13/cobra"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server"
"k8s.io/klog/v2"
ocmfeature "open-cluster-management.io/api/feature"
"open-cluster-management.io/ocm/pkg/features"

"open-cluster-management.io/multicluster-controlplane/pkg/agent"
mcfeature "open-cluster-management.io/multicluster-controlplane/pkg/feature"
)

func init() {
utilruntime.Must(features.SpokeMutableFeatureGate.Add(ocmfeature.DefaultSpokeRegistrationFeatureGates))
utilruntime.Must(features.SpokeMutableFeatureGate.Add(ocmfeature.DefaultSpokeWorkFeatureGates))
utilruntime.Must(features.SpokeMutableFeatureGate.Add(mcfeature.DefaultControlPlaneAgentFeatureGates))
}

func NewAgent() *cobra.Command {
Expand All @@ -40,7 +41,14 @@ func NewAgent() *cobra.Command {
ctx, terminate := context.WithCancel(shutdownCtx)
defer terminate()

if err := agentOptions.RunAgent(ctx); err != nil {
go func() {
klog.Info("starting the controlplane agent")
if err := agentOptions.RunAgent(ctx); err != nil {
klog.Fatalf("failed to run agent, %v", err)
}
}()

if err := agentOptions.RunAddOns(ctx); err != nil {
return err
}

Expand Down
14 changes: 8 additions & 6 deletions pkg/cmd/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,27 @@ package controller

import (
"fmt"

"github.com/spf13/cobra"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
ocmfeature "open-cluster-management.io/api/feature"
"open-cluster-management.io/ocm/pkg/features"

"open-cluster-management.io/multicluster-controlplane/pkg/servers"
"open-cluster-management.io/multicluster-controlplane/pkg/servers/options"

genericapiserver "k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
cliflag "k8s.io/component-base/cli/flag"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/version/verflag"
ocmfeature "open-cluster-management.io/api/feature"
"open-cluster-management.io/ocm/pkg/features"

mcfeature "open-cluster-management.io/multicluster-controlplane/pkg/feature"
"open-cluster-management.io/multicluster-controlplane/pkg/servers"
"open-cluster-management.io/multicluster-controlplane/pkg/servers/options"
)

func init() {
utilruntime.Must(features.HubMutableFeatureGate.Add(ocmfeature.DefaultHubWorkFeatureGates))
utilruntime.Must(features.HubMutableFeatureGate.Add(ocmfeature.DefaultHubRegistrationFeatureGates))
utilruntime.Must(features.HubMutableFeatureGate.Add(ocmfeature.DefaultHubAddonManagerFeatureGates))
utilruntime.Must(features.HubMutableFeatureGate.Add(mcfeature.DefaultControlPlaneFeatureGates))
}

func NewController() *cobra.Command {
Expand Down
19 changes: 19 additions & 0 deletions pkg/controllers/addons/managedserviceaccount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// Copyright Contributors to the Open Cluster Management project

package addons

import (
"context"

"open-cluster-management.io/managed-serviceaccount/pkg/addon/commoncontroller"

ctrl "sigs.k8s.io/controller-runtime"
)

func SetupManagedServiceAccountWithManager(ctx context.Context, mgr ctrl.Manager) error {
ctrl := commoncontroller.NewEphemeralIdentityReconciler(mgr.GetCache(), mgr.GetClient())
if err := ctrl.SetupWithManager(mgr); err != nil {
return err
}
return nil
}
2 changes: 2 additions & 0 deletions pkg/controllers/bootstrap/crds.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ var baseCRDs = []string{
"crds/0000_02_clusters.open-cluster-management.io_placements.crd.yaml",
"crds/0000_03_clusters.open-cluster-management.io_placementdecisions.crd.yaml",
"crds/0000_05_clusters.open-cluster-management.io_addonplacementscores.crd.yaml",
"crds/0000_06_authentication.open-cluster-management.io_managedserviceaccounts_crd.yaml",
}

var ocmCRDs = []string{
Expand All @@ -50,6 +51,7 @@ var ocmCRDs = []string{
"placementdecisions.cluster.open-cluster-management.io",
"placements.cluster.open-cluster-management.io",
"addonplacementscores.cluster.open-cluster-management.io",
"managedserviceaccounts.authentication.open-cluster-management.io",
}

var (
Expand Down
Loading

0 comments on commit 1001b0e

Please sign in to comment.