common d.SprintHeader() functions for humans
Signed-off-by: Jeff Carr <jcarr@wit.com>
This commit is contained in:
parent
26cd0f7709
commit
c1d86fc324
|
@ -9,7 +9,6 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.wit.com/lib/gui/shell"
|
|
||||||
pb "go.wit.com/lib/protobuf/virtbuf"
|
pb "go.wit.com/lib/protobuf/virtbuf"
|
||||||
"go.wit.com/log"
|
"go.wit.com/log"
|
||||||
)
|
)
|
||||||
|
@ -35,7 +34,7 @@ func create(w http.ResponseWriter, r *http.Request) (string, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.StartState = pb.DropletState_OFF
|
d.StartState = pb.DropletState_OFF
|
||||||
d.Current.State = pb.DropletState_OFF
|
d.SetState(pb.DropletState_OFF)
|
||||||
d.Memory = 2048 * 1024 * 1024
|
d.Memory = 2048 * 1024 * 1024
|
||||||
d.Cpus = 2
|
d.Cpus = 2
|
||||||
|
|
||||||
|
@ -106,9 +105,9 @@ func startDroplet(d *pb.Droplet) (string, error) {
|
||||||
// how long has the cluster been stable?
|
// how long has the cluster been stable?
|
||||||
// wait until it is stable. use this to throttle droplet starts
|
// wait until it is stable. use this to throttle droplet starts
|
||||||
dur := time.Since(me.unstable)
|
dur := time.Since(me.unstable)
|
||||||
result = fmt.Sprintln("should start droplet", name, "here. grid stable for:", shell.FormatDuration(dur))
|
result = fmt.Sprintln("should start droplet", name, "here. grid stable for:", pb.FormatDuration(dur))
|
||||||
if dur < me.unstableTimeout {
|
if dur < me.unstableTimeout {
|
||||||
tmp := shell.FormatDuration(me.unstableTimeout)
|
tmp := pb.FormatDuration(me.unstableTimeout)
|
||||||
result += "grid is still too unstable (unstable timeout = " + tmp + ")"
|
result += "grid is still too unstable (unstable timeout = " + tmp + ")"
|
||||||
return result, errors.New("grid is still unstable")
|
return result, errors.New("grid is still unstable")
|
||||||
}
|
}
|
||||||
|
|
42
dump.go
42
dump.go
|
@ -3,10 +3,8 @@ package main
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.wit.com/lib/gui/shell"
|
|
||||||
pb "go.wit.com/lib/protobuf/virtbuf"
|
pb "go.wit.com/lib/protobuf/virtbuf"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -36,29 +34,33 @@ func dumpDroplets(w http.ResponseWriter, full bool) {
|
||||||
d := loop.Droplet()
|
d := loop.Droplet()
|
||||||
fmt.Println(w, "Droplet UUID:", d.Uuid)
|
fmt.Println(w, "Droplet UUID:", d.Uuid)
|
||||||
|
|
||||||
var macs []string
|
|
||||||
for _, n := range d.Networks {
|
|
||||||
macs = append(macs, n.Mac)
|
|
||||||
}
|
|
||||||
|
|
||||||
// this line in golang could replace 80 lines of COBOL
|
// this line in golang could replace 80 lines of COBOL
|
||||||
header := fmt.Sprintf("%-3s %20s %-8s", d.Current.State, strings.Join(macs, " "), d.Current.Hypervisor)
|
header := d.SprintDumpHeader()
|
||||||
|
|
||||||
|
// check if this is a locally defined libvirt domain that needs to be imported
|
||||||
|
if d.LocalOnly != "" {
|
||||||
|
header += "(local)"
|
||||||
|
}
|
||||||
|
header += d.Hostname
|
||||||
|
|
||||||
|
if d.Current.State == pb.DropletState_ON {
|
||||||
|
// everything is as it should be with this vm
|
||||||
|
fmt.Fprintln(w, header)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if d.StartState == pb.DropletState_ON {
|
||||||
|
// this is supposed to be ON and needs to be turned on
|
||||||
|
fmt.Fprintln(w, header, "(should be on). todo: start() here")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if full {
|
||||||
var filenames string
|
var filenames string
|
||||||
for _, disk := range d.Disks {
|
for _, disk := range d.Disks {
|
||||||
filenames += disk.Filename
|
filenames += disk.Filename
|
||||||
}
|
}
|
||||||
|
|
||||||
if d.Current.State == pb.DropletState_ON {
|
// this needs to be turned on
|
||||||
fmt.Fprintln(w, header, d.Hostname)
|
fmt.Fprintln(w, header, filenames)
|
||||||
continue
|
|
||||||
}
|
|
||||||
if d.StartState == pb.DropletState_ON {
|
|
||||||
fmt.Fprintln(w, header, d.Hostname, "(should be on)")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if full {
|
|
||||||
fmt.Fprintln(w, header, d.Hostname, filenames)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -70,11 +72,11 @@ func dumpHypervisors(w http.ResponseWriter) {
|
||||||
for _, h := range me.hypers {
|
for _, h := range me.hypers {
|
||||||
// lastpoll time.Time // the last time the hypervisor polled
|
// lastpoll time.Time // the last time the hypervisor polled
|
||||||
dur := time.Since(h.lastpoll)
|
dur := time.Since(h.lastpoll)
|
||||||
tmp := shell.FormatDuration(dur)
|
tmp := pb.FormatDuration(dur)
|
||||||
fmt.Fprintln(w, h.pb.Hostname, "killcount =", h.killcount, "lastpoll:", tmp)
|
fmt.Fprintln(w, h.pb.Hostname, "killcount =", h.killcount, "lastpoll:", tmp)
|
||||||
for name, t := range h.lastDroplets {
|
for name, t := range h.lastDroplets {
|
||||||
dur := time.Since(t)
|
dur := time.Since(t)
|
||||||
tmp := shell.FormatDuration(dur)
|
tmp := pb.FormatDuration(dur)
|
||||||
totalDroplets += 1
|
totalDroplets += 1
|
||||||
d := me.cluster.FindDropletByName(name)
|
d := me.cluster.FindDropletByName(name)
|
||||||
if d == nil {
|
if d == nil {
|
||||||
|
|
|
@ -41,7 +41,9 @@ func importDomain(w http.ResponseWriter, r *http.Request) (string, error) {
|
||||||
fmt.Fprintln(w, result)
|
fmt.Fprintln(w, result)
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
result := start + " local ready to import from hypervisor"
|
result := start + "about to attempt import "
|
||||||
|
result += "(" + d.LocalOnly + ")"
|
||||||
|
result += " " + d.Hostname
|
||||||
log.Log(WARN, result)
|
log.Log(WARN, result)
|
||||||
fmt.Fprintln(w, result)
|
fmt.Fprintln(w, result)
|
||||||
return result, nil
|
return result, nil
|
||||||
|
|
2
main.go
2
main.go
|
@ -70,7 +70,7 @@ func main() {
|
||||||
if d.Current == nil {
|
if d.Current == nil {
|
||||||
d.Current = new(pb.Current)
|
d.Current = new(pb.Current)
|
||||||
}
|
}
|
||||||
d.Current.State = pb.DropletState_OFF
|
d.SetState(pb.DropletState_OFF)
|
||||||
log.Info("droplet", d.Hostname)
|
log.Info("droplet", d.Hostname)
|
||||||
}
|
}
|
||||||
hmm := "pihole.wit.com"
|
hmm := "pihole.wit.com"
|
||||||
|
|
30
poll.go
30
poll.go
|
@ -33,16 +33,16 @@ func (h *HyperT) pollHypervisor() {
|
||||||
}
|
}
|
||||||
state := fields[0]
|
state := fields[0]
|
||||||
name := fields[1]
|
name := fields[1]
|
||||||
start := fmt.Sprintf("%-9s %-20.20s", h.pb.Hostname, name)
|
|
||||||
d := me.cluster.FindDropletByName(name)
|
d := me.cluster.FindDropletByName(name)
|
||||||
if d == nil {
|
if d == nil {
|
||||||
log.Log(WARN, start, "local defined domain")
|
log.Log(WARN, name, "local defined domain")
|
||||||
log.Log(WARN, start, "local Adding new entry with AddDropletLocal()")
|
log.Log(WARN, name, "local Adding new entry with AddDropletLocal()")
|
||||||
log.Log(WARN, start, "local Adding new entry with AddDropletLocal()")
|
log.Log(WARN, name, "local Adding new entry with AddDropletLocal()")
|
||||||
log.Log(WARN, start, "local Adding new entry with AddDropletLocal()")
|
log.Log(WARN, name, "local Adding new entry with AddDropletLocal()")
|
||||||
me.cluster.AddDropletLocal(name, h.pb.Hostname)
|
me.cluster.AddDropletLocal(name, h.pb.Hostname)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
start := d.SprintHeader()
|
||||||
h.lastDroplets[name] = time.Now()
|
h.lastDroplets[name] = time.Now()
|
||||||
if state == "OFF" {
|
if state == "OFF" {
|
||||||
if d.LocalOnly == "" {
|
if d.LocalOnly == "" {
|
||||||
|
@ -57,7 +57,7 @@ func (h *HyperT) pollHypervisor() {
|
||||||
log.Log(POLL, start, "STATE:", state, "rest:", fields[2:])
|
log.Log(POLL, start, "STATE:", state, "rest:", fields[2:])
|
||||||
|
|
||||||
// update the status to ON
|
// update the status to ON
|
||||||
d.Current.State = pb.DropletState_ON
|
d.SetState(pb.DropletState_ON)
|
||||||
|
|
||||||
// set the LastPoll time to now
|
// set the LastPoll time to now
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
@ -106,20 +106,20 @@ func (h *HyperT) pollHypervisor() {
|
||||||
}
|
}
|
||||||
// everthing below here is dumb and needs to be rethought
|
// everthing below here is dumb and needs to be rethought
|
||||||
if d.Current.State != pb.DropletState_UNKNOWN {
|
if d.Current.State != pb.DropletState_UNKNOWN {
|
||||||
d.Current.State = pb.DropletState_UNKNOWN
|
d.SetState(pb.DropletState_UNKNOWN)
|
||||||
log.Info("set state UNKNOWN here", name)
|
log.Info("set state UNKNOWN here", name)
|
||||||
}
|
}
|
||||||
if d.Current.State == pb.DropletState_UNKNOWN {
|
if d.Current.State == pb.DropletState_UNKNOWN {
|
||||||
if dur > time.Minute*2 {
|
if dur > time.Minute*2 {
|
||||||
// what this means is the droplet probably wasn't migrated or the migrate failed
|
// what this means is the droplet probably wasn't migrated or the migrate failed
|
||||||
// where should this be checked? the status needs to be changed to OFF
|
// where should this be checked? the status needs to be changed to OFF
|
||||||
s := shell.FormatDuration(dur)
|
s := pb.FormatDuration(dur)
|
||||||
log.Info("UNKNOWN state for more than 2 minutes (clearing out ?)", name, s)
|
log.Info("UNKNOWN state for more than 2 minutes (clearing out ?)", name, s)
|
||||||
|
|
||||||
// it might be safe to set the status to OFF here. not really. this poll needs
|
// it might be safe to set the status to OFF here. not really. this poll needs
|
||||||
// to be moved somewhere else. there needs to be a new goroutine not tied to the
|
// to be moved somewhere else. there needs to be a new goroutine not tied to the
|
||||||
// hypervisor
|
// hypervisor
|
||||||
d.Current.State = pb.DropletState_OFF
|
d.SetState(pb.DropletState_OFF)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -158,13 +158,13 @@ func uptimeCheck() (bool, string) {
|
||||||
unknownList = append(unknownList, d.Hostname)
|
unknownList = append(unknownList, d.Hostname)
|
||||||
case pb.DropletState_ON:
|
case pb.DropletState_ON:
|
||||||
if dur > me.missingDropletTimeout {
|
if dur > me.missingDropletTimeout {
|
||||||
log.Info("GOOD STATE MISSING", d.Hostname, hname, shell.FormatDuration(dur))
|
log.Info("GOOD STATE MISSING", d.Hostname, hname, pb.FormatDuration(dur))
|
||||||
good = false
|
good = false
|
||||||
d.Current.State = pb.DropletState_UNKNOWN
|
d.SetState(pb.DropletState_UNKNOWN)
|
||||||
failed += 1
|
failed += 1
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
l := shell.FormatDuration(dur)
|
l := pb.FormatDuration(dur)
|
||||||
if l == "" {
|
if l == "" {
|
||||||
log.Info("DUR IS EMPTY", dur)
|
log.Info("DUR IS EMPTY", dur)
|
||||||
missing = append(missing, d)
|
missing = append(missing, d)
|
||||||
|
@ -173,12 +173,12 @@ func uptimeCheck() (bool, string) {
|
||||||
working += 1
|
working += 1
|
||||||
// log.Info("GOOD STATE ON", d.Hostname, d.hname, "dur =", l)
|
// log.Info("GOOD STATE ON", d.Hostname, d.hname, "dur =", l)
|
||||||
case pb.DropletState_OFF:
|
case pb.DropletState_OFF:
|
||||||
log.Info("OFF STATE", d.StartState, d.Hostname, hname, shell.FormatDuration(dur))
|
log.Info("OFF STATE", d.StartState, d.Hostname, hname, pb.FormatDuration(dur))
|
||||||
good = false
|
good = false
|
||||||
failed += 1
|
failed += 1
|
||||||
// missing = append(missing, d)
|
// missing = append(missing, d)
|
||||||
default:
|
default:
|
||||||
log.Info("WTF STATE", d.StartState, d.Hostname, hname, "Current.State =", d.Current.State, shell.FormatDuration(dur))
|
log.Info("WTF STATE", d.StartState, d.Hostname, hname, "Current.State =", d.Current.State, pb.FormatDuration(dur))
|
||||||
good = false
|
good = false
|
||||||
failed += 1
|
failed += 1
|
||||||
missing = append(missing, d)
|
missing = append(missing, d)
|
||||||
|
@ -202,7 +202,7 @@ func uptimeCheck() (bool, string) {
|
||||||
summary += "(killcount=" + fmt.Sprintf("%d", me.killcount) + ")"
|
summary += "(killcount=" + fmt.Sprintf("%d", me.killcount) + ")"
|
||||||
}
|
}
|
||||||
last := time.Since(me.unstable)
|
last := time.Since(me.unstable)
|
||||||
s := strings.TrimSpace(shell.FormatDuration(last))
|
s := strings.TrimSpace(pb.FormatDuration(last))
|
||||||
if last > me.clusterStableDuration {
|
if last > me.clusterStableDuration {
|
||||||
// the cluster has not been stable for 10 seconds
|
// the cluster has not been stable for 10 seconds
|
||||||
summary += "(stable=" + s + ")"
|
summary += "(stable=" + s + ")"
|
||||||
|
|
5
start.go
5
start.go
|
@ -10,7 +10,6 @@ import (
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.wit.com/lib/gui/shell"
|
|
||||||
pb "go.wit.com/lib/protobuf/virtbuf"
|
pb "go.wit.com/lib/protobuf/virtbuf"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,9 +17,9 @@ func isClusterStable() (string, error) {
|
||||||
// how long has the cluster been stable?
|
// how long has the cluster been stable?
|
||||||
// wait until it is stable. use this to throttle droplet starts
|
// wait until it is stable. use this to throttle droplet starts
|
||||||
dur := time.Since(me.unstable)
|
dur := time.Since(me.unstable)
|
||||||
good := fmt.Sprintln("trying to start droplet here. grid stable for: ", shell.FormatDuration(dur))
|
good := fmt.Sprintln("trying to start droplet here. grid stable for: ", pb.FormatDuration(dur))
|
||||||
if dur < me.unstableTimeout {
|
if dur < me.unstableTimeout {
|
||||||
tmp := shell.FormatDuration(me.unstableTimeout)
|
tmp := pb.FormatDuration(me.unstableTimeout)
|
||||||
err := "grid is still too unstable (unstable timeout = " + tmp + ")\n"
|
err := "grid is still too unstable (unstable timeout = " + tmp + ")\n"
|
||||||
return good + err, errors.New(err)
|
return good + err, errors.New(err)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue