reject start and create early if grid unstable

Signed-off-by: Jeff Carr <jcarr@wit.com>
This commit is contained in:
Jeff Carr 2024-10-30 19:38:12 -05:00
parent eacf3b8bef
commit 41673f3170
2 changed files with 27 additions and 10 deletions

View File

@ -42,6 +42,15 @@ func create(w http.ResponseWriter, r *http.Request) (string, error) {
log.Info("Got msg:", string(msg)) log.Info("Got msg:", string(msg))
log.Info("hostname =", d.Hostname) log.Info("hostname =", d.Hostname)
name := d.Hostname name := d.Hostname
// don't continue past here if the grid is unstable anyway
// because this will add the droplet to cluster.Droplets
if s, err := isClusterStable(); err != nil {
log.Info(s)
fmt.Fprintln(w, s)
return s, err
}
tmpd := findDroplet(name) tmpd := findDroplet(name)
if tmpd != nil { if tmpd != nil {
result := "create error: Droplet " + name + " is already defined" result := "create error: Droplet " + name + " is already defined"

View File

@ -14,11 +14,29 @@ import (
pb "go.wit.com/lib/protobuf/virtbuf" pb "go.wit.com/lib/protobuf/virtbuf"
) )
func isClusterStable() (string, error) {
// how long has the cluster been stable?
// wait until it is stable. use this to throttle droplet starts
dur := time.Since(me.unstable)
good := fmt.Sprintln("trying to start droplet here. grid stable for: ", shell.FormatDuration(dur))
if dur < me.unstableTimeout {
tmp := shell.FormatDuration(me.unstableTimeout)
err := "grid is still too unstable (unstable timeout = " + tmp + ")\n"
return good + err, errors.New(err)
}
return good, nil
}
// for now, because sometimes this should write to stdout and // for now, because sometimes this should write to stdout and
// sometimes to http socket, it returns a string // sometimes to http socket, it returns a string
func Start(name string) (string, error) { func Start(name string) (string, error) {
var result string var result string
if s, err := isClusterStable(); err != nil {
result += s
return result, err
}
// lookup the droplet by name // lookup the droplet by name
d := findDroplet(name) d := findDroplet(name)
if d == nil { if d == nil {
@ -38,16 +56,6 @@ func Start(name string) (string, error) {
return result, errors.New(result) return result, errors.New(result)
} }
// how long has the cluster been stable?
// wait until it is stable. use this to throttle droplet starts
dur := time.Since(me.unstable)
result = fmt.Sprintln("should start droplet", name, "here. grid stable for:", shell.FormatDuration(dur))
if dur < me.unstableTimeout {
tmp := shell.FormatDuration(me.unstableTimeout)
result += "grid is still too unstable (unstable timeout = " + tmp + ")"
return result, errors.New("grid is still unstable")
}
// make the list of hypervisors that are active and can start new droplets // make the list of hypervisors that are active and can start new droplets
var pool []*HyperT var pool []*HyperT
for _, h := range me.hypers { for _, h := range me.hypers {