add node affinity (#94)

* add node affinity

Signed-off-by: soulseen <sunzhu@yunify.com>

* update e2e test

Signed-off-by: soulseen <sunzhu@yunify.com>

config/crds/devops_v1alpha1_s2ibuilder.yaml

@@ -235,6 +235,14 @@ spec:
                   description: LayeredBuild describes if this is build which layered
                     scripts and sources on top of BuilderImage.
                   type: boolean
+                nodeAffinityKey:
+                  description: The key of Node Affinity.
+                  type: string
+                nodeAffinityValues:
+                  description: The values of Node Affinity.
+                  items:
+                    type: string
+                  type: array
                 outputImageName:
                   description: OutputImageName is a result image name without tag,
                     default is latest. tag will append to ImageName in the end
@@ -366,6 +374,9 @@ spec:
                 tag:
                   description: Tag is a result image tag name.
                   type: string
+                taintKey:
+                  description: The name of taint.
+                  type: string
                 usage:
                   description: Usage allows for properly shortcircuiting s2i logic
                     when `s2i usage` is invoked

config/samples/devops_v1alpha1_s2ibuilder.yaml

@@ -15,4 +15,7 @@ spec:
     tag: latest
     builderPullPolicy: if-not-present
     revisionId: "staging"
+    nodeAffinityKey: "affinityKey1"
+    nodeAffinityValues: ["affinityValue1"]
+    taintKey: "taint1"
     buildVolumes: ["s2i_python35_cache:/opt/app-root/lib"]

deploy/s2ioperator.yaml

@@ -235,6 +235,14 @@ spec:
                   description: LayeredBuild describes if this is build which layered
                     scripts and sources on top of BuilderImage.
                   type: boolean
+                nodeAffinityKey:
+                  description: The key of Node Affinity.
+                  type: string
+                nodeAffinityValue:
+                  description: The values of Node Affinity.
+                  items:
+                    type: string
+                  type: array
                 outputImageName:
                   description: OutputImageName is a result image name without tag,
                     default is latest. tag will append to ImageName in the end
@@ -366,6 +374,9 @@ spec:
                 tag:
                   description: Tag is a result image tag name.
                   type: string
+                taintKey:
+                  description: The name of taint.
+                  type: string
                 usage:
                   description: Usage allows for properly shortcircuiting s2i logic
                     when `s2i usage` is invoked

pkg/apis/devops/v1alpha1/s2ibuilder_types.go

@@ -406,6 +406,15 @@ type S2iConfig struct {
 
 	// The RevisionId is a branch name or a SHA-1 hash of every important thing about the commit
 	RevisionId string `json:"revisionId,omitempty"`
+
+	// The name of taint.
+	TaintKey string `json:"taintKey,omitempty"`
+
+	// The key of Node Affinity.
+	NodeAffinityKey string `json:"nodeAffinityKey,omitempty"`
+
+	// The values of Node Affinity.
+	NodeAffinityValues []string `json:"nodeAffinityValues,omitempty"`
 }
 
 type UserDefineTemplate struct {

pkg/controller/s2irun/ksbuilder_jobs.go

@@ -16,7 +16,10 @@ import (
 )
 
 const (
-	ConfigDataKey = "data"
+	ConfigDataKey     = "data"
+	TaintKey          = "kubesphere.io/ci"
+	NodeAffinityKey   = "node-role.kubesphere.io/worker"
+	NodeAffinityValue = "ci"
 )
 
 func (r *ReconcileS2iRun) NewConfigMap(instance *devopsv1alpha1.S2iRun, config devopsv1alpha1.S2iConfig, template *devopsv1alpha1.UserDefineTemplate) (*corev1.ConfigMap, error) {
@@ -250,6 +253,69 @@ func setJobLabelAnnotations(instance *devopsv1alpha1.S2iRun, config devopsv1alph
 		job.Annotations[devopsv1alpha1.DescriptionAnnotations] = description
 	}
 }
+func setJobLabelandToleration(job *batchv1.Job, config devopsv1alpha1.S2iConfig) {
+	var taintKey = TaintKey
+	var nodeAffinityKey = NodeAffinityKey
+	var nodeAffinityValues = []string{NodeAffinityValue}
+
+	if config.TaintKey != "" {
+		taintKey = config.TaintKey
+	}
+
+	if config.NodeAffinityKey != "" {
+		nodeAffinityKey = config.NodeAffinityKey
+	}
+
+	if config.NodeAffinityValues != nil {
+		nodeAffinityValues = config.NodeAffinityValues
+	}
+
+	tolerationNoSchedule := corev1.Toleration{
+		Key:      taintKey,
+		Operator: corev1.TolerationOpExists,
+		Effect:   corev1.TaintEffectNoSchedule,
+	}
+
+	tolerationPreferNoSchedule := corev1.Toleration{
+		Key:      taintKey,
+		Operator: corev1.TolerationOpExists,
+		Effect:   corev1.TaintEffectPreferNoSchedule,
+	}
+
+	tolerations := []corev1.Toleration{
+		tolerationNoSchedule,
+		tolerationPreferNoSchedule,
+	}
+
+	nodeSelectorRequirements := corev1.NodeSelectorRequirement{
+		Key:      nodeAffinityKey,
+		Operator: corev1.NodeSelectorOpIn,
+		Values:   nodeAffinityValues,
+	}
+
+	preferredSchedulingTerms := corev1.PreferredSchedulingTerm{
+		Weight: 1,
+		Preference: corev1.NodeSelectorTerm{
+			MatchExpressions: []corev1.NodeSelectorRequirement{
+				nodeSelectorRequirements,
+			},
+		},
+	}
+
+	preferredDuringSchedulingIgnoredDuringExecutions := []corev1.PreferredSchedulingTerm{
+		preferredSchedulingTerms,
+	}
+
+	affinity := &corev1.Affinity{
+		NodeAffinity: &corev1.NodeAffinity{
+			PreferredDuringSchedulingIgnoredDuringExecution: preferredDuringSchedulingIgnoredDuringExecutions,
+		},
+	}
+
+	job.Spec.Template.Spec.Affinity = affinity
+	job.Spec.Template.Spec.Tolerations = tolerations
+
+}
 func setConfigMapLabelAnnotations(instance *devopsv1alpha1.S2iRun, config devopsv1alpha1.S2iConfig, template *devopsv1alpha1.UserDefineTemplate, cm *corev1.ConfigMap) {
 	description := ""
 	imageName := GetNewImageName(instance, config)

pkg/controller/s2irun/s2irun_controller.go

@@ -195,6 +195,7 @@ func (r *ReconcileS2iRun) Reconcile(request reconcile.Request) (reconcile.Result
 		return reconcile.Result{}, err
 	}
 	setJobLabelAnnotations(instance, *builder.Spec.Config, builder.Spec.FromTemplate, job)
+	setJobLabelandToleration(job, *builder.Spec.Config)
 	found := &batchv1.Job{}
 	err = r.Get(context.TODO(), types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, found)
 	if err != nil && k8serror.IsNotFound(err) {

test/e2e/e2e_test.go

@@ -21,7 +21,12 @@ import (
 
 var _ = Describe("", func() {
 
-	const timeout = time.Second * 25
+	const (
+		timeout           = time.Second * 25
+		TaintKey          = "kubesphere.io/ci"
+		NodeAffinityKey   = "node-role.kubesphere.io/worker"
+		NodeAffinityValue = "ci"
+	)
 	It("Should work well when using exactly the runtimeimage example yamls", func() {
 		//create a s2ibuilder
 		s2ibuilder := &devopsv1alpha1.S2iBuilder{}
@@ -63,6 +68,9 @@ var _ = Describe("", func() {
 		job := &batchv1.Job{}
 		Eventually(func() error { return testClient.Get(context.TODO(), depKey, job) }, timeout, time.Second).
 			Should(Succeed())
+
+		res := checkAnffinitTaint(job, NodeAffinityKey, NodeAffinityValue, TaintKey)
+		Expect(res).To(Equal(true))
 
 		//for our example, the status must be successful
 		Eventually(func() error {
@@ -134,6 +142,9 @@ var _ = Describe("", func() {
 		Eventually(func() error { return testClient.Get(context.TODO(), depKey, job) }, timeout, time.Second).
 			Should(Succeed())
 
+		res := checkAnffinitTaint(job, s2ibuilder.Spec.Config.NodeAffinityKey, s2ibuilder.Spec.Config.NodeAffinityValues[0], s2ibuilder.Spec.Config.TaintKey)
+		Expect(res).To(Equal(true))
+
 		//for our example, the status must be successful
 		Eventually(func() error {
 			err = testClient.Get(context.TODO(), depKey, job)
@@ -643,3 +654,25 @@ var _ = Describe("", func() {
 		}, timeout, time.Second).Should(BeTrue())
 	})
 })
+
+func checkAnffinitTaint(job *batchv1.Job, nodeAffinityKey string, nodeAffinityValue string, taintKey string) (bool) {
+	if job.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key != nodeAffinityKey {
+		fmt.Println("Check nodeAffinityKey error.")
+		fmt.Println(job.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key, nodeAffinityKey)
+		return false
+
+	}
+	if job.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Values[0] != nodeAffinityValue {
+		fmt.Println("Check nodeAffinityValue error.")
+		fmt.Println(job.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Values[0], nodeAffinityValue)
+		return false
+	}
+
+	for _, toleration := range job.Spec.Template.Spec.Tolerations {
+		if toleration.Key == taintKey {
+			return true
+		}
+	}
+	fmt.Println("Check toleration error.")
+	return false
+}