healthcheck.go

package flywheel

import (
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/ec2"
)

// Status keeps track of the status
type Status uint

// Different state of the systems
const (
	STOPPED Status = iota
	STARTING
	STARTED
	STOPPING
	UNHEALTHY
)

// String - Working with integer statuses is mostly better, but it's
// occasionally necessary to output the status name.
func (n Status) String() string {
	switch n {
	case STOPPED:
		return "STOPPED"
	case STARTING:
		return "STARTING"
	case STARTED:
		return "STARTED"
	case STOPPING:
		return "STOPPING"
	case UNHEALTHY:
		return "UNHEALTHY"
	default:
		return "INVALIDSTATUS"
	}
}

// HealthWatcher - Check the status of the instances. Currently checks if they are "ready"; all
// stopped or all started. Will need to be extended to determine actual status.
func (fw *Flywheel) HealthWatcher(out chan<- Status) {
	out <- fw.CheckAll()

	ticker := time.NewTicker(fw.hcInterval)
	for {
		select {
		case <-ticker.C:
			out <- fw.CheckAll()
		}
	}
}

// CheckAll - check asg/instance state
// TODO - add more information what is unhealthy
func (fw *Flywheel) CheckAll() Status {
	health := make(map[string]int)

	err := fw.checkInstances(health)
	if err != nil {
		log.Print(err)
		return UNHEALTHY
	}

	err = fw.checkStoppedAutoScalingGroups(health)
	if err != nil {
		log.Print(err)
		return UNHEALTHY
	}

	_, terminated := health["terminated"]
	_, starting := health["pending"]
	_, stopping := health["stopping"]
	_, shutting := health["shutting-down"]
	_, running := health["running"]
	_, stopped := health["stopped"]

	switch {
	case starting && (stopping || shutting):
		log.Print("Unhealthy: Mix of starting and stopping resources")
		return UNHEALTHY

	case running && stopped:
		log.Print("Unhealthy: Mix of running and stopped resources")
		return UNHEALTHY

	case terminated:
		log.Print("Instance terminated, manual intervention required")
		return UNHEALTHY

	case starting:
		return STARTING

	case stopping, shutting:
		return STOPPING

	case running:
		return STARTED

	case stopped:
		return STOPPED

	default:
		log.Printf("Unhealthy: %v", health)
		return UNHEALTHY
	}
}

func (fw *Flywheel) checkInstances(health map[string]int) error {
	if len(fw.config.Instances) == 0 {
		return nil
	}
	resp, err := fw.ec2.DescribeInstances(
		&ec2.DescribeInstancesInput{
			InstanceIds: fw.config.AwsInstances(),
		},
	)
	if err != nil {
		log.Print(err)
		return err
	}

	for _, reservation := range resp.Reservations {
		for _, instance := range reservation.Instances {
			state := *instance.State.Name
			health[state] = health[state] + 1
		}
	}

	return nil
}

func (fw *Flywheel) checkStoppedAutoScalingGroups(health map[string]int) error {
	var err error
	var awsGroupNames []*string

	if len(fw.config.AutoScaling.Stop) == 0 {
		return nil
	}

	for _, groupName := range fw.config.AutoScaling.Stop {
		awsGroupNames = append(awsGroupNames, &groupName)
	}

	resp, err := fw.autoscaling.DescribeAutoScalingGroups(
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: awsGroupNames,
		},
	)
	if err != nil {
		return err
	}

	for _, group := range resp.AutoScalingGroups {
		running := true

		instanceIds := []*string{}
		for _, instance := range group.Instances {
			instanceIds = append(instanceIds, instance.InstanceId)
		}

		iResp, err := fw.ec2.DescribeInstances(
			&ec2.DescribeInstancesInput{
				InstanceIds: instanceIds,
			},
		)
		if err != nil {
			return err
		}

		for _, reservation := range iResp.Reservations {
			for _, instance := range reservation.Instances {
				state := *instance.State.Name
				health[state] = health[state] + 1
				running = running && *instance.State.Name == "running"
			}
		}

		// if all instances are running and ASG is suspended
		// resume the group
		if running && len(group.SuspendedProcesses) > 0 {
			for _, instance := range group.Instances {
				fw.autoscaling.SetInstanceHealth(
					&autoscaling.SetInstanceHealthInput{
						InstanceId:   instance.InstanceId,
						HealthStatus: aws.String("Healthy"),
					},
				)
			}

			_, err = fw.autoscaling.ResumeProcesses(
				&autoscaling.ScalingProcessQuery{
					AutoScalingGroupName: group.AutoScalingGroupName,
				},
			)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// NOT USED ?
func (fw *Flywheel) checkTerminatedAutoScalingGroups(health map[string]int) error {
	var err error
	var awsGroupNames []*string

	for groupName := range fw.config.AutoScaling.Terminate {
		awsGroupNames = append(awsGroupNames, &groupName)
	}

	resp, err := fw.autoscaling.DescribeAutoScalingGroups(
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: awsGroupNames,
		},
	)
	if err != nil {
		log.Print(err)
		return err
	}

	for _, group := range resp.AutoScalingGroups {
		if *group.MaxSize == 0 {
			if len(group.Instances) == 0 {
				health["stopped"]++
			} else {
				health["stopping"]++
			}
			continue
		}

		healthy := true
		for _, instance := range group.Instances {
			if *instance.HealthStatus != "Healthy" {
				healthy = false
				break
			}
		}

		if healthy {
			health["running"]++
			if len(group.SuspendedProcesses) > 0 {
				_, err = fw.autoscaling.ResumeProcesses(
					&autoscaling.ScalingProcessQuery{
						AutoScalingGroupName: group.AutoScalingGroupName,
					},
				)
				if err != nil {
					return err
				}
			}
		} else {
			health["starting"]++
		}
	}

	return nil
}