diff --git a/Makefile b/Makefile index b268238..8279f1d 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ start-pihole.wit.com: build ./virtigo --libvirt /tmp/pihole.wit.com.xml start-pihole.wit.com-http: - curl --silent http://localhost:8080/start?start=pihole.wit.com + curl --silent http://localhost:8080/start?hostname=pihole.wit.com old-start-all-droplets: curl --silent http://localhost:8080/start?start=git.wit.org diff --git a/argv.go b/argv.go index f3abcdb..71b9d1d 100644 --- a/argv.go +++ b/argv.go @@ -60,6 +60,7 @@ func init() { INFO = log.NewFlag("INFO", false, full, short, "general virtigo") POLL = log.NewFlag("POLL", false, full, short, "virtigo polling") - SPEW = log.NewFlag("SPEW", true, full, short, "bad things") + WARN = log.NewFlag("WARN", true, full, short, "bad things") + SPEW = log.NewFlag("SPEW", true, full, short, "dump everything") EVENT = log.NewFlag("EVENT", true, full, short, "hypeprvisor/droplet events") } diff --git a/event.go b/event.go index 0b4549a..0a8ea63 100644 --- a/event.go +++ b/event.go @@ -142,6 +142,12 @@ func Start(name string) (bool, string) { n := a + rand.Intn(b-a) result += fmt.Sprintln("pool has", len(pool), "members", "rand =", n) h := pool[n] + + // send the search directories to the hypervisor + result += fmt.Sprintln("h.sendDirs() HERE") + result += fmt.Sprintln("h.sendDirs() HERE") + h.sendDirs() + startbool, startresult := h.start(d) return startbool, result + startresult } diff --git a/http.go b/http.go index 684c9a8..d00117b 100644 --- a/http.go +++ b/http.go @@ -17,11 +17,11 @@ func cleanURL(url string) string { } func okHandler(w http.ResponseWriter, r *http.Request) { - var tmp string - tmp = cleanURL(r.URL.Path) + var route string + route = cleanURL(r.URL.Path) // is the cluster running what it should? - if tmp == "/droplets" { + if route == "/droplets" { for _, d := range me.cluster.Droplets { var msg string if d.StartState == pb.DropletState_ON { @@ -40,7 +40,7 @@ func okHandler(w http.ResponseWriter, r *http.Request) { } // show only what droplets should be running but are missing - if tmp == "/missing" { + if route == "/missing" { var count int var missing int for _, d := range me.cluster.Droplets { @@ -63,48 +63,56 @@ func okHandler(w http.ResponseWriter, r *http.Request) { return } - if tmp == "/favicon.ico" { + if route == "/favicon.ico" { // w.Header().Set("Content-Type", "image/svg+xml") w.Header().Set("Content-Type", "image/png") writeFile(w, "ipv6.png") return } - if tmp == "/goReference.svg" { + if route == "/goReference.svg" { w.Header().Set("Content-Type", "image/svg+xml") writeFile(w, "goReference.svg") return } - if tmp == "/dumplibvirtxml" { + if route == "/dumplibvirtxml" { virtigoxml.DumpLibvirtxmlDomainNames() return } - if tmp == "/uptime" { - b, s := clusterHealthy() - if b { - log.Info("Handling URL:", tmp, "cluster is ok", s) + if route == "/uptime" { + ok, s := clusterHealthy() + if ok { + log.Info("Handling URL:", route, "cluster is ok", s) fmt.Fprintln(w, s) } else { - log.Info("Handling URL:", tmp, "cluster is not right yet", s) + log.Info("Handling URL:", route, "cluster is not right yet", s) fmt.Fprintln(w, s) } return } - if tmp == "/start" { - start := r.URL.Query().Get("start") + if route == "/start" { + hostname := r.URL.Query().Get("hostname") + if hostname == "" { + log.Warn("start failed. hostname is blank", cleanURL(r.URL.Path)) + fmt.Fprintln(w, "start failed. hostname is blank", cleanURL(r.URL.Path)) + return + } + log.Warn("hostname is", hostname) + fmt.Fprintln(w, "hostname is", hostname) + // log.Warn("Handling URL:", tmp, "start droplet", start) - b, result := Start(start) + b, result := Start(hostname) log.Warn("Start returned =", b, "result =", result) fmt.Fprintln(w, "Start() returned", b) fmt.Fprintln(w, "result:", result) return } - log.Warn("BAD URL =", tmp) - fmt.Fprintln(w, "BAD URL tmp =", tmp) + log.Warn("BAD URL =", route) + fmt.Fprintln(w, "BAD URL tmp =", route) } // write a file out to the http socket diff --git a/main.go b/main.go index a5ef467..a36c0bc 100644 --- a/main.go +++ b/main.go @@ -43,8 +43,13 @@ func main() { me.changed = false // me.dmap = make(map[*pb.Droplet]*DropletT) me.hmap = make(map[*pb.Hypervisor]*HyperT) + + // how long a droplet can be missing until it's declared dead me.unstableTimeout = 17 * time.Second + // how long the cluster must be stable before new droplets can be started + me.clusterStableDuration = 37 * time.Second + // read in the config file me.cluster = new(pb.Cluster) if err := me.cluster.ConfigLoad(); err != nil { @@ -66,13 +71,13 @@ func main() { var newEvents []*pb.Event - // sanity check the droplets - virtigoxml.CheckDroplets(me.cluster, false) - newe := virtigoxml.CheckDiskFilenames(me.cluster) + // sanity check the cluster & droplets + ValidateDroplets(me.cluster, false) + newe := ValidateDiskFilenames(me.cluster) for _, e := range newe { newEvents = append(newEvents, e) } - virtigoxml.CheckUniqueFilenames(me.cluster) + ValidateUniqueFilenames(me.cluster) for _, filename := range argv.Xml { domcfg, err := virtigoxml.ReadXml(filename) diff --git a/poll.go b/poll.go index c60debe..c0ae08f 100644 --- a/poll.go +++ b/poll.go @@ -97,7 +97,7 @@ func clusterHealthy() (bool, string) { var total int var working int var failed int - var missing int + var missing []*pb.Droplet var unknown int var unknownList []string @@ -121,6 +121,7 @@ func clusterHealthy() (bool, string) { log.Info("BAD STATE", d.StartState, d.Hostname, hname, "CurrentState =", d.CurrentState, shell.FormatDuration(dur)) good = false failed += 1 + missing = append(missing, d) } else { dur := time.Since(d.LastPoll.AsTime()) // Calculate the elapsed time if dur > time.Minute { @@ -133,7 +134,7 @@ func clusterHealthy() (bool, string) { l := shell.FormatDuration(dur) if l == "" { log.Info("DUR IS EMPTY", dur) - missing += 1 + missing = append(missing, d) continue } working += 1 @@ -143,8 +144,8 @@ func clusterHealthy() (bool, string) { var summary string = "(" summary += fmt.Sprintf("total = %d ", total) summary += fmt.Sprintf("working = %d ", working) - if missing > 0 { - summary += fmt.Sprintf("missing = %d ", missing) + if len(missing) > 0 { + summary += fmt.Sprintf("missing = %d ", len(missing)) } if unknown > 0 { summary += fmt.Sprintf("unknown = %d ", unknown, unknownList) @@ -158,11 +159,14 @@ func clusterHealthy() (bool, string) { summary += "(killcount=" + fmt.Sprintf("%d", me.killcount) + ")" } last := time.Since(me.unstable) - if last > 133*time.Second { + if last > me.clusterStableDuration { // the cluster has not been stable for 10 seconds s := strings.TrimSpace(shell.FormatDuration(last)) summary += "(stable=" + s + ")" } + for _, d := range missing { + summary += fmt.Sprint("\nmissing droplet: ", d.Hostname, " current state ", d.CurrentState) + } if good { return good, "GOOD=true " + summary } diff --git a/structs.go b/structs.go index 8365629..81d4d00 100644 --- a/structs.go +++ b/structs.go @@ -28,7 +28,8 @@ type virtigoT struct { killcount int unstable time.Time // the last time the cluster was incorrect changed bool - unstableTimeout time.Duration + unstableTimeout time.Duration // how long a droplet can be missing until it's declared dead + clusterStableDuration time.Duration // how long the cluster must be stable before new droplets can be started } // the stuff that is needed for a hypervisor diff --git a/validate.go b/validate.go new file mode 100644 index 0000000..5343c34 --- /dev/null +++ b/validate.go @@ -0,0 +1,221 @@ +package main + +/* + validate / sanity check / consistancy check the data + + here is some code to do smart things like: + + * check mac addresses are unique + * check uuid's are unique + * double check filenames are unique + * return a unique mac address + * return a unique uuid + +*/ + +import ( + "errors" + "os" + "path/filepath" + + "github.com/google/uuid" + + pb "go.wit.com/lib/protobuf/virtbuf" + "go.wit.com/log" +) + +// will make sure the mac address is unique +func ValidateUniqueMac(cluster *pb.Cluster, mac string) bool { + for _, d := range cluster.Droplets { + for _, n := range d.Networks { + if n.Mac == mac { + log.Info("duplicate MAC", n.Mac, "in droplet", d.Hostname) + return false + } + } + } + return true +} + +// records all the known paths. this should go in the protobuf +func addClusterFilepath(cluster *pb.Cluster, dir string) *pb.Event { + var found bool = false + var e *pb.Event + for _, d := range cluster.Dirs { + if d == dir { + // found dir + found = true + break + } + } + if !found { + if dir != "." { + // make a new Add Event + e = pb.NewAddEvent(nil, "Add Cluster Directory", dir) + cluster.Dirs = append(cluster.Dirs, dir) + } + } + return e +} + +// returns the droplet using a filename +func lookupFilename(cluster *pb.Cluster, filename string) *pb.Droplet { + filebase := filepath.Base(filename) + + for _, d := range cluster.Droplets { + for _, disk := range d.Disks { + if filebase == disk.Filename { + return d + } + } + } + return nil +} + +func InsertFilename(cluster *pb.Cluster, d *pb.Droplet, filename string) (*pb.Event, error) { + dupd := lookupFilename(cluster, filename) + if dupd != nil { + log.Info("file", filename, "already on droplet", dupd.Hostname) + log.Info("file", filename, "on new droplet", d.Hostname) + if os.Getenv("VIRTIGO_IGNORE_DISKDUP") == "" { + log.Info("duplicate disk names (--xml-ignore-disk to ignore)") + return nil, errors.New("duplicate disk names") + } else { + log.Info("ignore duplicate disk names (--xml-ignore-disk=true)") + } + } + filebase := filepath.Base(filename) + dir := filepath.Dir(filename) + for _, disk := range d.Disks { + if disk.Filename == filebase { + log.Info("already have disk", filename) + return nil, nil + } + } + // make a new Add Event + e := d.NewChangeEvent("Add Disk", "", filename) + + // add the disk protobuf entry + var disk *pb.Disk + disk = new(pb.Disk) + disk.Filename = filebase + disk.Filepath = dir + d.Disks = append(d.Disks, disk) + log.Info("New filename", filebase, dir) + return e, nil +} + +func ValidateUniqueFilenames(cluster *pb.Cluster) bool { + var ok bool = true + var disks map[string]string + disks = make(map[string]string) + + for _, d := range cluster.Droplets { + for _, disk := range d.Disks { + filename := disk.Filename + addClusterFilepath(cluster, disk.Filepath) + if _, ok := disks[filename]; ok { + /* + if argv.IgnDisk { + log.Info("ignore dup disk", filename, disks[filename], d.Hostname) + } else { + } + */ + log.Info("file", filename, "on droplet", disks[filename]) + log.Info("file", filename, "on droplet", d.Hostname) + log.Info("duplicate disk names (--xml-ignore-disk to ignore)") + ok = false + } + disks[filename] = d.Hostname + } + } + if ok { + log.Println("validated okay: no duplicate disk images") + } + return ok +} + +func ValidateDiskFilenames(cluster *pb.Cluster) []*pb.Event { + var alle []*pb.Event + + for _, d := range cluster.Droplets { + for _, disk := range d.Disks { + filename := disk.Filename + filebase := filepath.Base(filename) + dir := filepath.Dir(filename) + addClusterFilepath(cluster, dir) + if disk.Filename != filebase { + // update filename + e := d.NewChangeEvent("Disk.Filename", disk.Filename, filebase) + alle = append(alle, e) + disk.Filename = filebase + } + if dir == "." { + continue + } + if dir == "" { + continue + } + if disk.Filepath != dir { + // update filename + e := d.NewChangeEvent("Disk.Filepath", disk.Filepath, dir) + alle = append(alle, e) + disk.Filepath = dir + } + } + } + return alle +} + +// this doesn't run often +func ValidateDroplets(cluster *pb.Cluster, dump bool) bool { + // uuid map to check for duplicates + var umap map[string]string + umap = make(map[string]string) + + // mac address map to check for duplicates + var macs map[string]string + macs = make(map[string]string) + + for _, d := range cluster.Droplets { + // Generate a new UUID + if d.Uuid == "" { + u := uuid.New() + d.Uuid = u.String() + } + + // seconds, ok := timeZone[tz]; ok { + if _, ok := umap[d.Uuid]; ok { + // UUID already exists + log.Info("duplicate UUID", d.Uuid, umap[d.Uuid]) + log.Info("duplicate UUID", d.Uuid, d.Hostname) + os.Exit(-1) + } + umap[d.Uuid] = d.Hostname + + for _, n := range d.Networks { + // log.Println("network:", n.Mac, d.Uuid, d.Hostname) + if _, ok := macs[n.Mac]; ok { + // UUID already exists + log.Info("duplicate MAC", n.Mac, macs[n.Mac], umap[macs[n.Mac]]) + log.Info("duplicate MAC", n.Mac, d.Hostname) + os.Exit(-1) + } + macs[n.Mac] = d.Uuid + } + } + log.Println("validated okay: no duplicate MAC addr") + log.Println("validated okay: no duplicate UUID") + + if dump { + for u, hostname := range umap { + log.Println("uuid:", u, "hostname:", hostname) + } + + for mac, uuid := range macs { + log.Println("mac:", mac, "uuid", uuid, "hostname:", umap[uuid]) + } + } + + return false +}