Skip to content

Commit

Permalink
Added prometheus rule creation function
Browse files Browse the repository at this point in the history
  • Loading branch information
obrienrobert committed Mar 31, 2020
1 parent 0179289 commit a437d4a
Showing 1 changed file with 54 additions and 4 deletions.
58 changes: 54 additions & 4 deletions pkg/resources/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ package resources
import (
"context"
"fmt"

"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"strings"

"github.com/integr8ly/integreatly-operator/pkg/apis/integreatly/v1alpha1"
Expand All @@ -15,6 +19,8 @@ import (
k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
)

const alertFor = "5m"

// CreatePostgresAvailabilityAlert creates a PrometheusRule alert to watch for the availability
// of a Postgres instance
func CreatePostgresAvailabilityAlert(ctx context.Context, client k8sclient.Client, inst *v1alpha1.RHMI, cr *crov1.Postgres) (*prometheusv1.PrometheusRule, error) {
Expand All @@ -37,7 +43,7 @@ func CreatePostgresAvailabilityAlert(ctx context.Context, client k8sclient.Clien
}

// create the rule
pr, err := croResources.ReconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
pr, err := reconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -65,7 +71,7 @@ func CreatePostgresConnectivityAlert(ctx context.Context, client k8sclient.Clien
}

// create the rule
pr, err := croResources.ReconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
pr, err := reconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -94,7 +100,7 @@ func CreateRedisAvailabilityAlert(ctx context.Context, client k8sclient.Client,
}

// create the rule
pr, err := croResources.ReconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
pr, err := reconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -122,9 +128,53 @@ func CreateRedisConnectivityAlert(ctx context.Context, client k8sclient.Client,
}

// create the rule
pr, err := croResources.ReconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
pr, err := reconcilePrometheusRule(ctx, client, ruleName, cr.Namespace, alertName, alertDescription, alertExp, labels)
if err != nil {
return nil, err
}
return pr, nil
}

// reconcilePrometheusRule will create a PrometheusRule object
func reconcilePrometheusRule(ctx context.Context, client k8sclient.Client, ruleName, ns, alertName, desc string, alertExp intstr.IntOrString, labels map[string]string) (*prometheusv1.PrometheusRule, error) {
alertGroupName := alertName + "Group"
groups := []prometheusv1.RuleGroup{
{
Name: alertGroupName,
Rules: []prometheusv1.Rule{
{
Alert: alertName,
Expr: alertExp,
For: alertFor,
Labels: labels,
Annotations: map[string]string{
"description": desc,
},
},
},
},
}

rule := &prometheusv1.PrometheusRule{
ObjectMeta: metav1.ObjectMeta{
Name: ruleName,
Namespace: ns,
Labels: map[string]string{
"monitoring-key": "middleware",
},
},
Spec: prometheusv1.PrometheusRuleSpec{
Groups: groups,
},
}

// create or update the resource
_, err := controllerutil.CreateOrUpdate(ctx, client, rule, func() error {
return nil
})
if err != nil {
return nil, errors.Wrapf(err, "failed to reconcile prometheus rule request for %s", ruleName)
}

return rule, nil
}

0 comments on commit a437d4a

Please sign in to comment.