148 lines
4.3 KiB
Go
148 lines
4.3 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"time"
|
|
|
|
"go.wit.com/lib/gui/shell"
|
|
pb "go.wit.com/lib/protobuf/virtbuf"
|
|
"go.wit.com/log"
|
|
)
|
|
|
|
// restarts the virtigod daemon on a hypervisor via http
|
|
func (h *HyperT) RestartVirtigod() {
|
|
url := "http://" + h.pb.Hostname + ":2520/kill"
|
|
s := shell.Wget(url)
|
|
log.Info("EVENT RestartVirtigod", url, s)
|
|
h.lastpoll = time.Now()
|
|
h.killcount += 1
|
|
|
|
dur := time.Since(h.lastpoll) // Calculate the elapsed time
|
|
log.Info("KILLED DAEMON", h.pb.Hostname, shell.FormatDuration(dur), "curl", url)
|
|
me.killcount += 1
|
|
|
|
// mark the cluster as unstable so droplet starts can be throttled
|
|
me.unstable = time.Now()
|
|
}
|
|
|
|
// checks if the cluster is ready and stable
|
|
func clusterReady() (bool, string) {
|
|
last := time.Since(me.unstable)
|
|
if last > me.unstableTimeout {
|
|
// the cluster has not been stable for 133 seconds
|
|
log.Warn("clusterReady() is stable for ", shell.FormatDuration(me.unstableTimeout), " secs")
|
|
return true, fmt.Sprintln("clusterReady() is stable ", shell.FormatDuration(me.unstableTimeout), " secs")
|
|
}
|
|
log.Warn("clusterReady() is unstable for", shell.FormatDuration(last))
|
|
return false, "clusterReady() is unstable for " + shell.FormatDuration(last)
|
|
}
|
|
|
|
func dropletReady(d *pb.Droplet) (bool, string) {
|
|
if d.CurrentState == pb.DropletState_ON {
|
|
return false, "EVENT start droplet is already ON"
|
|
}
|
|
if d.Starts > 2 {
|
|
// reason := "EVENT start droplet has already been started " + d.starts + " times"
|
|
return false, fmt.Sprintln("EVENT start droplet has already been started ", d.Starts, " times")
|
|
}
|
|
return true, ""
|
|
}
|
|
|
|
// this must be bool in string because accumulated output is sometimes
|
|
// written to STDOUT, sometimes to http
|
|
func (h *HyperT) start(d *pb.Droplet) (bool, string) {
|
|
ready, result := clusterReady()
|
|
if !ready {
|
|
return false, result
|
|
}
|
|
ready, result = dropletReady(d)
|
|
if !ready {
|
|
return false, result
|
|
}
|
|
|
|
url := "http://" + h.pb.Hostname + ":2520/start?start=" + d.Hostname
|
|
var msg string
|
|
var data []byte
|
|
msg = d.FormatJSON()
|
|
data = []byte(msg) // Convert the string to []byte
|
|
req, err := httpPost(url, data)
|
|
if err != nil {
|
|
return false, fmt.Sprintln("error:", err)
|
|
}
|
|
log.Info("http post url:", url)
|
|
log.Info("http post data:", msg)
|
|
|
|
result = "EVENT start droplet url: " + url + "\n"
|
|
result += "EVENT start droplet response: " + string(req)
|
|
|
|
// increment the counter for a start attempt working
|
|
d.Starts += 1
|
|
|
|
// mark the cluster as unstable so droplet starts can be throttled
|
|
me.unstable = time.Now()
|
|
|
|
return true, result
|
|
}
|
|
|
|
func findDroplet(name string) *pb.Droplet {
|
|
for _, d := range me.cluster.Droplets {
|
|
if d.Hostname == name {
|
|
return d
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func Start(name string) (bool, string) {
|
|
var result string
|
|
|
|
d := findDroplet(name)
|
|
if d == nil {
|
|
result += "can't start unknown droplet: " + name
|
|
return false, result
|
|
}
|
|
|
|
if d.CurrentState == pb.DropletState_ON {
|
|
return false, "EVENT start droplet " + d.Hostname + " is already ON"
|
|
}
|
|
|
|
dur := time.Since(me.unstable) // how long has the cluster been stable?
|
|
result = fmt.Sprintln("should start droplet", name, "here. grid stable for:", shell.FormatDuration(dur))
|
|
if dur < me.unstableTimeout {
|
|
result += "grid is still too unstable"
|
|
return false, result
|
|
}
|
|
|
|
// make the list of hypervisors that are active and can start new droplets
|
|
var pool []*HyperT
|
|
for _, h := range me.hypers {
|
|
result += fmt.Sprintln("could start droplet on", name, "on", h.pb.Hostname, h.pb.Active)
|
|
if d.PreferredHypervisor == h.pb.Hostname {
|
|
// the config file says this droplet should run on this hypervisor
|
|
a, b := h.start(d)
|
|
return a, result + b
|
|
}
|
|
|
|
if h.pb.Active != true {
|
|
continue
|
|
}
|
|
pool = append(pool, h)
|
|
}
|
|
|
|
// left here as an example of how to actually do random numbers
|
|
// it's complete mathematical chaos. Randomness is simple when
|
|
// human interaction occurs -- which is exactly what happens most
|
|
// of the time. most random shit is bullshit. all you really need
|
|
// is exactly this to make sure the random functions work as they
|
|
// should. Probably, just use this everywhere in all cases. --jcarr
|
|
rand.Seed(time.Now().UnixNano())
|
|
a := 0
|
|
b := len(pool)
|
|
n := a + rand.Intn(b-a)
|
|
result += fmt.Sprintln("pool has", len(pool), "members", "rand =", n)
|
|
h := pool[n]
|
|
startbool, startresult := h.start(d)
|
|
return startbool, result + startresult
|
|
}
|