diff --git a/config/crd/bases/operator.victoriametrics.com_vmagents.yaml b/config/crd/bases/operator.victoriametrics.com_vmagents.yaml index 3b2177e3..5c755088 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmagents.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmagents.yaml @@ -544,7 +544,7 @@ spec: type: boolean type: object configMaps: - description: ConfigMaps is a list of ConfigMaps in the same namespace as the vmagent object, which shall be mounted into the vmagent Pods. will be mounted at path /etc/vmagent/configs + description: ConfigMaps is a list of ConfigMaps in the same namespace as the vmagent object, which shall be mounted into the vmagent Pods. will be mounted at path /etc/vm/configs items: type: string type: array @@ -1108,7 +1108,7 @@ spec: pattern: '[0-9]+(ms|s|m|h)' type: string secrets: - description: Secrets is a list of Secrets in the same namespace as the vmagent object, which shall be mounted into the vmagent Pods. will be mounted at path /etc/vmagent/secrets + description: Secrets is a list of Secrets in the same namespace as the vmagent object, which shall be mounted into the vmagent Pods. will be mounted at path /etc/vm/secrets items: type: string type: array diff --git a/config/crd/bases/operator.victoriametrics.com_vmalerts.yaml b/config/crd/bases/operator.victoriametrics.com_vmalerts.yaml index e76dfeb3..b763030e 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmalerts.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmalerts.yaml @@ -372,7 +372,7 @@ spec: type: object type: object configMaps: - description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMAlert object, which shall be mounted into the VMAlert Pods. The ConfigMaps are mounted into /etc/vmalert/configmaps/. + description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMAlert object, which shall be mounted into the VMAlert Pods. The ConfigMaps are mounted into /etc/vm/configs/. items: type: string type: array @@ -1242,7 +1242,7 @@ spec: type: object type: object secrets: - description: Secrets is a list of Secrets in the same namespace as the VMAlert object, which shall be mounted into the VMAlert Pods. The Secrets are mounted into /etc/vmalert/secrets/. + description: Secrets is a list of Secrets in the same namespace as the VMAlert object, which shall be mounted into the VMAlert Pods. The Secrets are mounted into /etc/vm/secrets/. items: type: string type: array diff --git a/config/crd/bases/operator.victoriametrics.com_vmclusters.yaml b/config/crd/bases/operator.victoriametrics.com_vmclusters.yaml index 4ffab43c..afda2460 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmclusters.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmclusters.yaml @@ -412,7 +412,7 @@ spec: type: object type: object configMaps: - description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The ConfigMaps are mounted into /etc/vmalert/configmaps/. + description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The ConfigMaps are mounted into /etc/vm/configs/. items: type: string type: array @@ -605,7 +605,7 @@ spec: description: SchedulerName - defines kubernetes scheduler name type: string secrets: - description: Secrets is a list of Secrets in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The Secrets are mounted into /etc/vmalert/secrets/. + description: Secrets is a list of Secrets in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The Secrets are mounted into /etc/vm/secrets/. items: type: string type: array @@ -1889,7 +1889,7 @@ spec: description: CacheMountPath allows to add cache persistent for VMSelect type: string configMaps: - description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The ConfigMaps are mounted into /etc/vm/configmaps/. + description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The ConfigMaps are mounted into /etc/vm/configs/. items: type: string type: array @@ -3547,7 +3547,7 @@ spec: type: object type: object configMaps: - description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The ConfigMaps are mounted into /etc/vmalert/configmaps/. + description: ConfigMaps is a list of ConfigMaps in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The ConfigMaps are mounted into /etc/vm/configs/. items: type: string type: array @@ -3740,7 +3740,7 @@ spec: description: SchedulerName - defines kubernetes scheduler name type: string secrets: - description: Secrets is a list of Secrets in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The Secrets are mounted into /etc/vmalert/secrets/. + description: Secrets is a list of Secrets in the same namespace as the VMSelect object, which shall be mounted into the VMSelect Pods. The Secrets are mounted into /etc/vm/secrets/. items: type: string type: array diff --git a/controllers/vmprobe_controller.go b/controllers/vmprobe_controller.go index 93a38353..6db8fb99 100644 --- a/controllers/vmprobe_controller.go +++ b/controllers/vmprobe_controller.go @@ -38,6 +38,7 @@ type VMProbeReconciler struct { BaseConf *config.BaseOperatorConf } +// Reconcile - syncs VMProbe // +kubebuilder:rbac:groups=operator.victoriametrics.com,resources=vmprobes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=operator.victoriametrics.com,resources=vmprobes/status,verbs=get;update;patch func (r *VMProbeReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { @@ -78,6 +79,7 @@ func (r *VMProbeReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } +// SetupWithManager - setups VMProbe manager func (r *VMProbeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&operatorv1beta1.VMProbe{}). diff --git a/controllers/vmprometheusconverter_controller.go b/controllers/vmprometheusconverter_controller.go index a0f13e78..72fb91c3 100644 --- a/controllers/vmprometheusconverter_controller.go +++ b/controllers/vmprometheusconverter_controller.go @@ -21,6 +21,33 @@ import ( "k8s.io/client-go/tools/cache" ) +const ( + // MetaMergeStrategyLabel merge strategy by default prefer prometheus meta labels + // but with annotation value added to VMObject: + // annotations: + // operator.victoriametrics.com/merge-api-strategy: prefer-victoriametrics + // metadata from VMObject will be preferred during merge + MetaMergeStrategyLabel = "operator.victoriametrics.com/merge-meta-strategy" + // MetaPreferVM - prefers VM object meta values, ignores prometheus + MetaPreferVM = "prefer-victoriametrics" + // MetaPreferProm - prefers prometheus + MetaPreferProm = "prefer-prometheus" + // MetaMergeLabelsVMPriority merges both label sets + // its not possible to remove values + MetaMergeLabelsVMPriority = "merge-victoriametrics-priority" + // MetaMergeLabelsPromPriority merges both label sets + // its not possible to remove values + MetaMergeLabelsPromPriority = "merge-prometheus-priority" + + // IgnoreConversionLabel this annotation disables updating of corresponding VMObject + // must be added to annotation of VMObject + // annotations: + // operator.victoriametrics.com/ignore-prometheus-updates: enabled + IgnoreConversionLabel = "operator.victoriametrics.com/ignore-prometheus-updates" + // IgnoreConversion - disables updates from prometheus api + IgnoreConversion = "enabled" +) + // ConverterController - watches for prometheus objects // and create VictoriaMetrics objects type ConverterController struct { @@ -212,8 +239,15 @@ func (c *ConverterController) UpdatePrometheusRule(old, new interface{}) { l.Error(err, "cannot get existing VMRule") return } - + if existingVMRule.Annotations[IgnoreConversionLabel] == IgnoreConversion { + l.Info("syncing for object was disabled by annotation", "annotation", IgnoreConversionLabel) + return + } existingVMRule.Spec = VMRule.Spec + metaMergeStrategy := getMetaMergeStrategy(existingVMRule.Annotations) + existingVMRule.Annotations = mergeLabelsWithStrategy(existingVMRule.Annotations, VMRule.Annotations, metaMergeStrategy) + existingVMRule.Labels = mergeLabelsWithStrategy(existingVMRule.Labels, VMRule.Labels, metaMergeStrategy) + err = c.vclient.Update(ctx, existingVMRule) if err != nil { l.Error(err, "cannot update VMRule") @@ -254,8 +288,16 @@ func (c *ConverterController) UpdateServiceMonitor(old, new interface{}) { l.Error(err, "cannot get existing vmServiceScrape") return } + + if existingVMServiceScrape.Annotations[IgnoreConversionLabel] == IgnoreConversion { + l.Info("syncing for object was disabled by annotation", "annotation", IgnoreConversionLabel) + return + } existingVMServiceScrape.Spec = vmServiceScrape.Spec + metaMergeStrategy := getMetaMergeStrategy(existingVMServiceScrape.Annotations) + existingVMServiceScrape.Annotations = mergeLabelsWithStrategy(existingVMServiceScrape.Annotations, vmServiceScrape.Annotations, metaMergeStrategy) + existingVMServiceScrape.Labels = mergeLabelsWithStrategy(existingVMServiceScrape.Labels, vmServiceScrape.Labels, metaMergeStrategy) err = c.vclient.Update(ctx, existingVMServiceScrape) if err != nil { l.Error(err, "cannot update") @@ -295,7 +337,16 @@ func (c *ConverterController) UpdatePodMonitor(old, new interface{}) { l.Error(err, "cannot get existing podMonitor") return } + if existingVMPodScrape.Annotations[IgnoreConversionLabel] == IgnoreConversion { + l.Info("syncing for object was disabled by annotation", "annotation", IgnoreConversionLabel) + return + } + existingVMPodScrape.Spec = podScrape.Spec + mergeStrategy := getMetaMergeStrategy(existingVMPodScrape.Annotations) + existingVMPodScrape.Annotations = mergeLabelsWithStrategy(existingVMPodScrape.Annotations, podScrape.Annotations, mergeStrategy) + existingVMPodScrape.Labels = mergeLabelsWithStrategy(existingVMPodScrape.Labels, podScrape.Labels, mergeStrategy) + err = c.vclient.Update(ctx, existingVMPodScrape) if err != nil { l.Error(err, "cannot update podScrape") @@ -305,6 +356,46 @@ func (c *ConverterController) UpdatePodMonitor(old, new interface{}) { } +// default merge strategy - prefer-prometheus +// old - from vm +// new - from prometheus +// by default new has priority +func mergeLabelsWithStrategy(old, new map[string]string, mergeStrategy string) map[string]string { + + switch mergeStrategy { + case MetaPreferVM: + return old + case MetaPreferProm: + return new + case MetaMergeLabelsVMPriority: + old, new = new, old + case MetaMergeLabelsPromPriority: + break + } + merged := make(map[string]string) + for k, v := range old { + merged[k] = v + } + for k, v := range new { + merged[k] = v + } + return merged +} + +// helper function - extracts meta merge strategy +// in the future we can introduce another merge strategies +func getMetaMergeStrategy(vmMeta map[string]string) string { + switch vmMeta[MetaMergeStrategyLabel] { + case MetaPreferVM: + return MetaPreferVM + case MetaMergeLabelsPromPriority: + return MetaMergeLabelsPromPriority + case MetaMergeLabelsVMPriority: + return MetaMergeLabelsVMPriority + } + return MetaPreferProm +} + // CreateProbe converts Probe to VMProbe func (c *ConverterController) CreateProbe(obj interface{}) { probe := obj.(*v1.Probe) @@ -336,6 +427,15 @@ func (c *ConverterController) UpdateProbe(old, new interface{}) { l.Error(err, "cannot get existing vmProbe") return } + if existingVMProbe.Annotations[IgnoreConversionLabel] == IgnoreConversion { + l.Info("syncing for object was disabled by annotation", "annotation", IgnoreConversionLabel) + return + } + + mergeStrategy := getMetaMergeStrategy(existingVMProbe.Annotations) + existingVMProbe.Annotations = mergeLabelsWithStrategy(existingVMProbe.Annotations, probeNew.Annotations, mergeStrategy) + existingVMProbe.Labels = mergeLabelsWithStrategy(existingVMProbe.Labels, probeNew.Labels, mergeStrategy) + existingVMProbe.Spec = vmProbe.Spec err = c.vclient.Update(ctx, existingVMProbe) if err != nil { diff --git a/controllers/vmprometheusconverter_controller_test.go b/controllers/vmprometheusconverter_controller_test.go new file mode 100644 index 00000000..19adecd3 --- /dev/null +++ b/controllers/vmprometheusconverter_controller_test.go @@ -0,0 +1,81 @@ +package controllers + +import ( + "reflect" + "testing" +) + +func Test_mergeLabelsWithStrategy(t *testing.T) { + type args struct { + old map[string]string + new map[string]string + mergeStrategy string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "delete not existing label", + args: args{ + old: map[string]string{"label1": "value1", "label2": "value2", "missinglabel": "value3"}, + new: map[string]string{"label1": "value1", "label2": "value4"}, + mergeStrategy: MetaPreferProm, + }, + want: map[string]string{"label1": "value1", "label2": "value4"}, + }, + { + name: "add new label", + args: args{ + old: map[string]string{"label1": "value1", "label2": "value2", "missinglabel": "value3"}, + new: map[string]string{"label1": "value1", "label2": "value4", "label5": "value10"}, + mergeStrategy: MetaPreferProm, + }, + want: map[string]string{"label1": "value1", "label2": "value4", "label5": "value10"}, + }, + { + name: "add new label with VM priority", + args: args{ + old: map[string]string{"label1": "value1", "label2": "value2", "label5": "value3"}, + new: map[string]string{"label1": "value1", "label2": "value4", "missinglabel": "value10"}, + mergeStrategy: MetaPreferVM, + }, + want: map[string]string{"label1": "value1", "label2": "value2", "label5": "value3"}, + }, + { + name: "remove all labels", + args: args{ + old: nil, + new: map[string]string{"label1": "value1", "label2": "value4", "missinglabel": "value10"}, + mergeStrategy: MetaPreferVM, + }, + want: nil, + }, + { + name: "remove keep old labels", + args: args{ + old: map[string]string{"label1": "value1", "label2": "value4"}, + new: nil, + mergeStrategy: MetaPreferVM, + }, + want: map[string]string{"label1": "value1", "label2": "value4"}, + }, + { + name: "merge all labels with VMPriority", + args: args{ + old: map[string]string{"label1": "value1", "label2": "value4"}, + new: map[string]string{"label1": "value2", "label2": "value4", "missinglabel": "value10"}, + mergeStrategy: MetaMergeLabelsVMPriority, + }, + want: map[string]string{"label1": "value1", "label2": "value4", "missinglabel": "value10"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := mergeLabelsWithStrategy(tt.args.old, tt.args.new, tt.args.mergeStrategy); !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeLabelsWithStrategy() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/docs/quick-start.MD b/docs/quick-start.MD index facf21a9..b6ff4c2f 100644 --- a/docs/quick-start.MD +++ b/docs/quick-start.MD @@ -1046,10 +1046,50 @@ VM_ENABLEDPROMETHEUSCONVERTER_SERVICESCRAPE=false VM_ENABLEDPROMETHEUSCONVERTER_PROMETHEUSRULE=false VM_ENABLEDPROMETHEUSCONVERTER_PROBE=false ``` - Otherwise, victoriametrics-operator would try to discover prometheus-operator API and convert it. + Conversion of api objects can be controlled by annotations, added to `VMObject`s, there are following annotations: + - `operator.victoriametrics.com/merge-meta-strategy` - it controls syncing of metadata labels and annotations between + `VMObject`s and `Prometheus` api objects during updates to `Prometheus` objects. By default, it has `prefer-prometheus`. + And annotations and labels will be used from `Prometheus` objects, manually set values will be dropped. + You can set it to `prefer-victoriametrics`. In this case all labels and annotations applied to `Prometheus` object + will be ignored and `VMObject` will use own values. + Two additional strategies annotations -`merge-victoriametrics-priority` and `merge-prometheus-priority` merges labelSets into one combined labelSet, with priority. + Example: +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + annotations: + meta.helm.sh/release-name: prometheus + operator.victoriametrics.com/merge-meta-strategy: prefer-victoriametrics + labels: + release: prometheus + name: prometheus-monitor +spec: + endpoints: [] +``` + +- `operator.victoriametrics.com/ignore-prometheus-updates` - it controls updates from Prometheus api objects. + By default, it set to `disabled`. You define it to `enabled` state and all updates from Prometheus api objects will be + ignored. +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + annotations: + meta.helm.sh/release-name: prometheus + operator.victoriametrics.com/ignore-prometheus-updates: enabled + labels: + release: prometheus + name: prometheus-monitor +spec: + endpoints: [] +``` + + + ## Expose the VMSingle API