135 lines
3.2 KiB
Go
135 lines
3.2 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.wit.com/lib/gui/shell"
|
|
"go.wit.com/log"
|
|
)
|
|
|
|
func (h *HyperT) pollHypervisor() {
|
|
url := "http://" + h.Hostname + ":2520/vms"
|
|
log.Log(POLL, "wget url =", url)
|
|
s := shell.Wget(url)
|
|
if s == nil {
|
|
return
|
|
}
|
|
var bytesSplice []byte
|
|
bytesSplice = s.Bytes()
|
|
// fmt.Fprintln(w, string(bytesSplice))
|
|
for _, line := range strings.Split(string(bytesSplice), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
continue
|
|
}
|
|
state := fields[0]
|
|
name := fields[1]
|
|
if state == "ON" {
|
|
log.Log(POLL, h.Hostname, "STATE:", state, "HOST:", name, "rest:", fields[2:])
|
|
d := findDroplet(name)
|
|
if d != nil {
|
|
log.Log(INFO, "ALREADY RECORDED", d.Hostname)
|
|
d.lastpoll = time.Now()
|
|
d.CurrentState = "ON"
|
|
// log.Info("ALREADY RECORDED", d.Hostname, d.lastpoll)
|
|
if d.hname == "" {
|
|
log.Log(EVENT, "DROPLET", d.Hostname, "PROBABLY WAS NEVER POLLED YET")
|
|
}
|
|
if d.hname != h.Hostname {
|
|
log.Log(EVENT, "DROPLET", d.Hostname, "MOVED FROM", d.hname, "TO", h.Hostname)
|
|
d.hname = h.Hostname
|
|
}
|
|
continue
|
|
}
|
|
// this is a new unknown droplet (not in the config file)
|
|
d = new(DropletT)
|
|
d.Hostname = name
|
|
d.hname = h.Hostname
|
|
d.lastpoll = time.Now()
|
|
d.CurrentState = "ON"
|
|
me.droplets = append(me.droplets, d)
|
|
log.Log(EVENT, name, "IS NEW. ADDED ON", h.Hostname)
|
|
}
|
|
}
|
|
h.lastpoll = time.Now()
|
|
h.killcount = 0 // poll worked. reset killcount
|
|
}
|
|
|
|
func findDroplet(name string) *DropletT {
|
|
for _, d := range me.droplets {
|
|
if d.Hostname == name {
|
|
return d
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// check the state of the cluster and return a string
|
|
// that is intended to be sent to an uptime monitor like Kuma
|
|
func clusterHealthy() (bool, string) {
|
|
var good bool = true
|
|
var total int
|
|
var working int
|
|
var failed int
|
|
var missing int
|
|
var unknown int
|
|
|
|
for _, d := range me.droplets {
|
|
total += 1
|
|
if d.State != "ON" {
|
|
continue
|
|
}
|
|
dur := time.Since(d.lastpoll) // Calculate the elapsed time
|
|
if d.CurrentState == "" {
|
|
// log.Info("SKIP. hostname has not been polled yet", d.Hostname, d.hname)
|
|
unknown += 1
|
|
continue
|
|
}
|
|
if d.CurrentState != "ON" {
|
|
log.Info("BAD STATE", d.State, d.Hostname, d.hname, "CurrentState =", d.CurrentState, shell.FormatDuration(dur))
|
|
good = false
|
|
failed += 1
|
|
} else {
|
|
dur := time.Since(d.lastpoll) // Calculate the elapsed time
|
|
if dur > time.Minute {
|
|
log.Info("GOOD STATE MISSING", d.Hostname, d.hname, shell.FormatDuration(dur))
|
|
good = false
|
|
d.CurrentState = "MISSING"
|
|
failed += 1
|
|
continue
|
|
}
|
|
l := shell.FormatDuration(dur)
|
|
if l == "" {
|
|
log.Info("DUR IS EMPTY", dur)
|
|
missing += 1
|
|
continue
|
|
}
|
|
working += 1
|
|
// log.Info("GOOD STATE ON", d.Hostname, d.hname, "dur =", l)
|
|
}
|
|
}
|
|
var summary string = "("
|
|
summary += fmt.Sprintf("total = %d ", total)
|
|
summary += fmt.Sprintf("working = %d ", working)
|
|
if missing > 0 {
|
|
summary += fmt.Sprintf("missing = %d ", missing)
|
|
}
|
|
if unknown > 0 {
|
|
summary += fmt.Sprintf("unknown = %d ", unknown)
|
|
}
|
|
if failed > 0 {
|
|
summary += fmt.Sprintf("failed = %d ", failed)
|
|
}
|
|
summary = strings.TrimSpace(summary)
|
|
summary += ")"
|
|
if good {
|
|
return good, "GOOD=true " + summary
|
|
}
|
|
return good, "GOOD=false " + summary
|
|
}
|