.gitignore

@@ -1,3 +1,5 @@
 go.*
+
 *.pb.go
-*.swp
+
+example/example

.protobuf

@@ -1,3 +0,0 @@
-droplet.proto
-event.proto
-hypervisor.proto

Makefile

@@ -3,15 +3,33 @@
 # cd ~/go/src/google.golang.org/protobuf/cmd/protoc-gen-go
 # go install
 
-all: proto goimports vet
+check-for-protoc-gen-go:
+	@if [ -f "/usr/bin/bin/protoc-gen-go" ]; then \
+		echo "the protoc-gen-go package is old in debian sid right now"; \
+		echo "for now, remove it"; \
+		apt remote proto-gen-go \
+		exit 1; \
+	fi
+	@if [ ! -f "$(HOME)/go/bin/protoc-gen-go" ]; then \
+		echo "you must build protoc-gen-go from google"; \
+		echo go-clone google.golang.org/protobuf; \
+		cd ~/go/src/google.golang.org/protobuf/cmd/protoc-gen-go/ && go install; \
+	fi
+	make all
 
-vet:
-	@GO111MODULE=off go vet
-	@echo this go library package builds okay
+all: droplet.pb.go hypervisor.pb.go event.pb.go experiments.pb.go
+	make -C example
+
+vet: lint
+	GO111MODULE=off go vet
+
+lint:
+	-buf lint droplet.proto
 
 # autofixes your import headers in your golang files
 goimports:
 	goimports -w *.go
+	make -C example goimports
 
 redomod:
 	rm -f go.*
@@ -21,20 +39,32 @@ redomod:
 clean:
 	rm -f *.pb.go
 	-rm -f go.*
-
-proto:droplet.pb.go hypervisor.pb.go event.pb.go cluster.pb.go
+	make -C example clean
 
 droplet.pb.go: droplet.proto
-	autogenpb --proto droplet.proto
+	# protoc --go_out=. droplet.proto
+	# This is switched over to use the new protoc-gen-go from   google.golang.org/protobuf/cmd/protoc-gen-go
+	# the debian one (2024/10/21) seems to be the older/original one from github.com/golang/protobuf/protoc-gen-go
+	cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/virtbuf \
+		--go_opt=Mdroplet.proto=go.wit.com/lib/protobuf/virtbuf \
+		droplet.proto
 
 hypervisor.pb.go: hypervisor.proto
-	autogenpb --proto hypervisor.proto
+	cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/virtbuf \
+		--go_opt=Mhypervisor.proto=go.wit.com/lib/protobuf/virtbuf \
+		hypervisor.proto
 
 event.pb.go: event.proto
-	autogenpb --proto event.proto
+	cd ~/go/src && protoc --go_out=. \
+		--proto_path=go.wit.com/lib/protobuf/virtbuf \
+		--go_opt=Mevent.proto=go.wit.com/lib/protobuf/virtbuf \
+		event.proto
 
-cluster.pb.go: cluster.proto
-	autogenpb --proto cluster.proto
+experiments.pb.go: experiments.proto
+	cd ~/go/src && protoc --go_out=. \
+		--proto_path=go.wit.com/lib/protobuf/virtbuf \
+		--go_opt=Mexperiments.proto=go.wit.com/lib/protobuf/virtbuf \
+		experiments.proto
 
 deps:
 	apt install golang-goprotobuf-dev

README.md

@@ -1,6 +1,13 @@
-This go library handles the protobuf files
-and various functions for virtigo.
+They are what to use for starting droplets with virsh,
+but for making a cluster or "home cloud" with virtigo
 
-You must build the protobuf files using autogenpb
+When possible & sensible, use the same variable names as libvirt
+
+virtigo needs to coordinate all the virtual machine
+information centrally. The information in the libvirt xml
+files are imported into these protobuf definitions and
+passed around as protobufs
+
+virtigo recreates the libvirt xml files and calls
+'virsh create'
 
-    go install go.wit.com/apps/autogenpb@latest

add.go

@@ -1,6 +1,7 @@
-package virtpb
+package virtbuf
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -8,8 +9,7 @@ import (
 	"go.wit.com/log"
 )
 
-/*
-func (c *OldCluster) InitDroplet(hostname string) (*Droplet, error) {
+func (c *NewCluster) InitDroplet(hostname string) (*Droplet, error) {
 	var d *Droplet
 	d = new(Droplet)
 	d.Current = new(Current)
@@ -28,13 +28,12 @@ func (c *OldCluster) InitDroplet(hostname string) (*Droplet, error) {
 	return d, nil
 }
 
-func (c *Cluster) appendDroplet(d *Droplet) {
+func (c *NewCluster) appendDroplet(d *Droplet) {
 	c.Lock()
 	defer c.Unlock()
 
 	c.d.Droplets = append(c.d.Droplets, d)
 }
-*/
 
 // can the json protobuf output use a string and have a type handler
 // to convert it back to int64?
@@ -50,10 +49,10 @@ func (x *Hypervisor) SetMemoryGB(gb int) {
 	x.Memory = int64(gb * 1024 * 1024 * 1024)
 }
 
-func (c *OldCluster) FindDropletByName(name string) *Droplet {
-	loop := c.d.All() // get the list of droplets
+func (c *NewCluster) FindDropletByName(name string) *Droplet {
+	loop := c.DropletsAll() // get the list of droplets
 	for loop.Scan() {
-		d := loop.Next()
+		d := loop.Droplet()
 		if d.Hostname == name {
 			return d
 		}
@@ -61,20 +60,7 @@ func (c *OldCluster) FindDropletByName(name string) *Droplet {
 	return nil
 }
 
-func (c *OldCluster) FindDropletByUuid(id string) *Droplet {
-	/*
-		log.Info("START FIND", id)
-		loop := c.d.All() // get the list of droplets
-		for loop.Scan() {
-			d := loop.Next()
-			log.Info("droplet:", d.Hostname, d.Uuid)
-		}
-		log.Info("END FIND", id)
-	*/
-	return c.d.FindByUuid(id)
-}
-
-func (c *OldCluster) FindHypervisorByName(name string) *Hypervisor {
+func (c *NewCluster) FindHypervisorByName(name string) *Hypervisor {
 	for _, h := range c.H.Hypervisors {
 		if h.Hostname == name {
 			return h
@@ -83,7 +69,7 @@ func (c *OldCluster) FindHypervisorByName(name string) *Hypervisor {
 	return nil
 }
 
-func (c *OldCluster) AddHypervisor(hostname string, cpus int, mem int) *Hypervisor {
+func (c *NewCluster) AddHypervisor(hostname string, cpus int, mem int) *Hypervisor {
 	h := c.FindHypervisorByName(hostname)
 	if h != nil {
 		return h
@@ -104,25 +90,15 @@ func (c *OldCluster) AddHypervisor(hostname string, cpus int, mem int) *Hypervis
 	return h
 }
 
-func (c *OldCluster) AddEvent(e *Event) {
+func (c *NewCluster) AddEvent(e *Event) {
 	c.e.Events = append(c.e.Events, e)
 }
 
-// creates a new droplet with default values
-func NewDefaultDroplet(hostname string) *Droplet {
-	id := uuid.New() // Generate a new UUID
-	d := &Droplet{
-		Uuid:     id.String(),
-		Hostname: hostname,
-		Cpus:     int64(2),
-	}
-	d.Memory = SetGB(2)
-	d.StartState = DropletState_OFF
-
-	return d
+func (c *NewCluster) AddDroplet(d *Droplet) {
+	c.d.Droplets = append(c.d.Droplets, d)
 }
 
-func (c *OldCluster) AddDropletSimple(uuid string, hostname string, cpus int, mem int) *Droplet {
+func (c *NewCluster) AddDropletSimple(uuid string, hostname string, cpus int, mem int) *Droplet {
 	d := c.FindDropletByName(hostname)
 	if d != nil {
 		return d
@@ -144,7 +120,7 @@ func (c *OldCluster) AddDropletSimple(uuid string, hostname string, cpus int, me
 }
 
 // This isn't for the marketing department
-func (c *OldCluster) AddDropletLocal(name string, hypername string) *Droplet {
+func (c *NewCluster) AddDropletLocal(name string, hypername string) *Droplet {
 	d := &Droplet{
 		Hostname: name,
 	}
@@ -162,10 +138,10 @@ func (c *OldCluster) AddDropletLocal(name string, hypername string) *Droplet {
 	return d
 }
 
-func (c *OldCluster) BlankFields() {
-	loop := c.d.All() // get the list of droplets
+func (c *NewCluster) BlankFields() {
+	loop := c.DropletsAll() // get the list of droplets
 	for loop.Scan() {
-		d := loop.Next()
+		d := loop.Droplet()
 		d.Current = nil
 	}
 }
@@ -174,7 +150,7 @@ func (epb *Events) AppendEvent(e *Event) {
 	epb.Events = append(epb.Events, e)
 }
 
-func (c *OldCluster) ClusterStable() (bool, string) {
+func (c *NewCluster) ClusterStable() (bool, string) {
 	last := time.Since(c.Unstable.AsTime())
 	if last > c.UnstableTimeout.AsDuration() {
 		// the cluster has not been stable for 133 seconds
@@ -186,7 +162,7 @@ func (c *OldCluster) ClusterStable() (bool, string) {
 }
 
 // check the cluster and droplet to make sure it's ready to start
-func (c *OldCluster) DropletReady(d *Droplet) (bool, string) {
+func (c *NewCluster) DropletReady(d *Droplet) (bool, string) {
 	if c == nil {
 		return false, "cluster == nil"
 	}

backupDir.go

@@ -1,4 +1,4 @@
-package virtpb
+package virtbuf
 
 // thank chatgpt for this because why. why write this if you can have it
 // kick this out in 30 seconds
@@ -14,7 +14,7 @@ func backupDir(srcDir string, destDir string) error {
 	// Create the destination directory
 	err := os.MkdirAll(destDir, os.ModePerm)
 	if err != nil {
-		log.Printf("Failed to create directory: %v\n", err)
+		log.Println("Failed to create directory: %v", err)
 		return err
 	}
 
@@ -41,7 +41,7 @@ func backupDir(srcDir string, destDir string) error {
 	})
 
 	if err != nil {
-		log.Printf("Failed to copy files: %v\n", err)
+		log.Println("Failed to copy files: %v", err)
 		return err
 	}
 	return nil

change.go

@@ -1,4 +1,4 @@
-package virtpb
+package virtbuf
 
 import (
 	// "reflect"
@@ -34,6 +34,7 @@ func convertToAnypb(x any) *anypb.Any {
 		return a
 	default:
 		log.Error(errors.New("convertToAnypb() unknown type"), "v =", v, "x =", x)
+		return nil
 	}
 	return nil
 }
@@ -64,6 +65,7 @@ func convertToString(x any) string {
 	default:
 		log.Info("convertToSTring() unknown type", v)
 		log.Error(errors.New("convertToSTring() unknown type"), "v =", v, "x =", x)
+		return ""
 	}
 	return ""
 }
@@ -73,7 +75,7 @@ func (d *Droplet) NewChangeEvent(fname string, origval any, newval any) *Event {
 	var e *Event
 	e = new(Event)
 
-	e.DropletName = d.Hostname
+	e.Droplet = d.Hostname
 	e.OrigVal = convertToString(origval)
 	e.NewVal = convertToString(newval)
 	e.FieldName = fname
@@ -99,12 +101,12 @@ func NewAddEvent(a any, fname string, newval any) *Event {
 	case *Droplet:
 		var d *Droplet
 		d = a.(*Droplet)
-		e.DropletName = d.Hostname
+		e.Droplet = d.Hostname
 	case nil:
-		e.DropletName = "<nil>"
+		e.Droplet = "<nil>"
 	default:
 		log.Info("newAddEvent() unknown type", v)
-		e.DropletName = "on something somewhere"
+		e.Droplet = "on something somewhere"
 	}
 
 	e.NewVal = convertToString(newval)
@@ -163,7 +165,7 @@ func (d *Droplet) SetState(newState DropletState) {
 }
 
 // records an event that the droplet changed state (aka turned on, turned off, etc)
-func (c *OldCluster) ChangeDropletState(d *Droplet, newState DropletState) error {
+func (c *NewCluster) ChangeDropletState(d *Droplet, newState DropletState) error {
 	if c == nil {
 		return errors.New("cluster is nil")
 	}
@@ -177,7 +179,7 @@ func (c *OldCluster) ChangeDropletState(d *Droplet, newState DropletState) error
 	var e *Event
 	e = new(Event)
 
-	e.DropletName = d.Hostname
+	e.Droplet = d.Hostname
 	e.OrigVal = convertToString(d.Current.State)
 	e.NewVal = convertToString(newState)
 	e.FieldName = "status"
@@ -190,7 +192,7 @@ func (c *OldCluster) ChangeDropletState(d *Droplet, newState DropletState) error
 }
 
 // records an event that the droplet migrated to another hypervisor
-func (c *OldCluster) DropletMoved(d *Droplet, newh *Hypervisor) error {
+func (c *NewCluster) DropletMoved(d *Droplet, newh *Hypervisor) error {
 	if c == nil {
 		return errors.New("cluster is nil")
 	}
@@ -209,7 +211,7 @@ func (c *OldCluster) DropletMoved(d *Droplet, newh *Hypervisor) error {
 	var e *Event
 	e = new(Event)
 
-	e.DropletName = d.Hostname
+	e.Droplet = d.Hostname
 	e.OrigVal = d.Current.Hypervisor
 	e.NewVal = newh.Hostname
 	e.FieldName = "droplet migrate"

cluster.proto

@@ -1,23 +0,0 @@
-syntax = "proto3";
-package virtpb;
-
-import "google/protobuf/timestamp.proto";
-import "droplet.proto";
-import "hypervisor.proto";
-import "event.proto";
-
-message Cluster {                                         // `autogenpb:marshal`
-        string                      uuid            = 1;  // `autogenpb:unique`
-        string                      name            = 2;
-        repeated string             URL             = 3;
-        google.protobuf.Timestamp   ctime           = 4;  // when the cluster was created
-        Droplets                    droplets        = 5;
-        Hypervisors                 hypervisors     = 6;
-        Events                      events          = 7;
-}
-
-message Clusters {                                        // `autogenpb:marshal`
-        string                      uuid            = 1;  // `autogenpb:uuid:57ddd763-75f6-4003-bf0e-8dd0f8a44044`
-        string                      version         = 2;  // `autogenpb:version:v0.0.1`
-        repeated Cluster            clusters        = 3;
-}

config.go

@@ -1,4 +1,4 @@
-package virtpb
+package virtbuf
 
 // functions to import and export the protobuf
 // data to and from config files
@@ -9,37 +9,14 @@ import (
 	"os"
 	"path/filepath"
 
-	"go.wit.com/log"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/encoding/prototext"
 	"google.golang.org/protobuf/reflect/protoreflect"
 )
 
-func (c *Cluster) ConfigSave() error {
-	name := c.Name
-	if name == "" {
-		name = c.Uuid
-	}
-	fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), name+".pb")
-	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
-	defer cfgfile.Close()
-	if err != nil {
-		fmt.Println("open config file :", err)
-		return err
-	}
-	log.Info("ConfigSave()", fullname)
-	data, err := c.Marshal()
-	if err != nil {
-		fmt.Println("cluster Marshal() err:", err)
-		return err
-	}
-	fmt.Fprintln(cfgfile, data)
-	return nil
-}
-
 // writes out the cluster information it seperate files
 // to make it humanly possible to hand edit things as needed
-func (c *OldCluster) ConfigSave() error {
+func (c *NewCluster) ConfigSave() error {
 	// try to backup the current cluster config files
 	if err := backupConfig(); err != nil {
 		return err
@@ -48,21 +25,31 @@ func (c *OldCluster) ConfigSave() error {
 	// make a new droplets struct
 	var dcopy *Droplets
 	dcopy = new(Droplets)
-	loop := c.d.All() // get the list of droplets
+	loop := c.DropletsAll() // get the list of droplets
 	for loop.Scan() {
-		d := loop.Next()
-		dcopy.Droplets = append(dcopy.Droplets, d)
+		d := loop.Droplet()
+		var newd Droplet
+		newd = *d
+		dcopy.Droplets = append(dcopy.Droplets, &newd)
 	}
 	// delete all the Current data so it's not put in the config file
 	for _, drop := range dcopy.Droplets {
 		drop.Current = nil
 	}
+	if err := ConfigWriteJSON(dcopy, "droplets.json"); err != nil {
+		fmt.Println("droplets.json write failed")
+		return err
+	}
 	if err := ConfigWriteTEXT(dcopy, "droplets.text"); err != nil {
 		fmt.Println("droplets.json write failed")
 		return err
 	}
 	c.configWriteDroplets()
 
+	if err := ConfigWriteJSON(c.H, "hypervisors.json"); err != nil {
+		fmt.Println("hypervisors.json write failed")
+		return err
+	}
 	if err := ConfigWriteTEXT(c.H, "hypervisors.text"); err != nil {
 		fmt.Println("hypervisors.json write failed")
 		return err
@@ -79,27 +66,27 @@ func (c *OldCluster) ConfigSave() error {
 	return nil
 }
 
-func (c *OldCluster) ConfigLoad() error {
+func (c *NewCluster) ConfigLoad() error {
 	if c == nil {
 		return errors.New("It's not safe to run ConfigLoad() on a nil cluster")
 	}
 
-	if data, err := loadFile("droplets.text"); err == nil {
-		if err = prototext.Unmarshal(data, c.d); err != nil {
-			fmt.Println("broken droplets.text config file")
+	if data, err := loadFile("droplets.json"); err == nil {
+		if err = protojson.Unmarshal(data, c.d); err != nil {
+			fmt.Println("broken droplets.json config file")
 			return err
 		}
 	} else {
 		return err
 	}
 
-	if data, err := loadFile("hypervisors.text"); err == nil {
-		if err = prototext.Unmarshal(data, c.H); err != nil {
-			fmt.Println("broken hypervisors.text config file")
+	if data, err := loadFile("hypervisors.json"); err == nil {
+		if err = protojson.Unmarshal(data, c.H); err != nil {
+			fmt.Println("broken hypervisors.json config file")
 			return err
 		}
 	} else {
-		log.Warn("ERROR HERE IN Hypervisors")
+		fmt.Println("ERROR HERE IN Hypervisors")
 		return err
 	}
 
@@ -108,11 +95,8 @@ func (c *OldCluster) ConfigLoad() error {
 		// does it not stay allocated after this function ends?
 		c.e = new(Events)
 	}
-
 	if err := c.e.loadEvents(); err != nil {
-		// ignore events.pb since these should be sent elsewhere
-		log.Warn("Events failed to load, ignoring:", err)
-		return nil
+		return err
 	}
 	return nil
 }
@@ -160,7 +144,7 @@ func loadFile(filename string) ([]byte, error) {
 
 func ConfigWriteJSON(a any, filename string) error {
 	fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), filename)
-	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE, 0666)
 	defer cfgfile.Close()
 	if err != nil {
 		fmt.Println("open config file :", err)
@@ -175,26 +159,9 @@ func ConfigWriteJSON(a any, filename string) error {
 	return nil
 }
 
-func (c *OldCluster) configWriteDroplets() error {
-	fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), "droplets.new.text")
-	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
-	defer cfgfile.Close()
-	if err != nil {
-		fmt.Println("open config file :", err)
-		return err
-	}
-	loop := c.d.All() // get the list of droplets
-	for loop.Scan() {
-		d := loop.Next()
-		text := prototext.Format(d)
-		fmt.Fprintln(cfgfile, text)
-	}
-	return nil
-}
-
 func ConfigWriteTEXT(a any, filename string) error {
 	fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), filename)
-	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE, 0666)
 	defer cfgfile.Close()
 	if err != nil {
 		fmt.Println("open config file :", err)
@@ -209,18 +176,22 @@ func ConfigWriteTEXT(a any, filename string) error {
 	return nil
 }
 
-func (c *Clusters) ConfigLoad() error {
-	if c == nil {
-		return errors.New("It's not safe to run ConfigLoad() on a nil cluster")
-	}
-
-	if data, err := loadFile("cluster.text"); err == nil {
-		if err = prototext.Unmarshal(data, c); err != nil {
-			fmt.Println("broken cluster.textconfig file")
+func (c *NewCluster) configWriteDroplets() error {
+	fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), "droplets.new.text")
+	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE, 0666)
+	defer cfgfile.Close()
+	if err != nil {
+		fmt.Println("open config file :", err)
 		return err
 	}
-	} else {
-		return err
+	loop := c.DropletsAll() // get the list of droplets
+	for loop.Scan() {
+		d := loop.Droplet()
+		var newd Droplet
+		newd = *d
+		newd.Current = nil
+		text := prototext.Format(&newd)
+		fmt.Fprintln(cfgfile, text)
 	}
 	return nil
 }

configBackup.go

@@ -1,4 +1,4 @@
-package virtpb
+package virtbuf
 
 // thank chatgpt for this because why. why write this if you can have it
 // kick this out in 30 seconds

droplet.proto

@@ -1,39 +1,37 @@
 syntax = "proto3";
-package virtpb;
+package virtbuf;
 
 import "google/protobuf/duration.proto"; // Import the well-known type for Timestamp
 import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
 
-// global settings for autogenpb `autogenpb:mutex`
-
-message Droplets {                                                // `autogenpb:marshal` `autogenpb:gui`
-        string                      uuid                    = 1;  // `autogenpb:uuid:d5d492e2-38d4-476b-86f3-f5abf01f9d6d`
-        string                      version                 = 2;  // `autogenpb:version:v0.0.1`
+message Droplets {
+	string  uuid	        = 1;    // I guess why not just have this on each file
+	string  version         = 2;    // maybe can be used for protobuf schema change violations
 	repeated Droplet droplets = 3;
 }
 
-message Droplet {                                                 // `autogenpb:marshal`
-        string                      uuid                    = 1;  // `autogenpb:unique` // should be unique across the cluster
-        string                      hostname                = 2;  // `autogenpb:unique` // should be unique and work in DNS
+message Droplet {
+	string          uuid	        = 1;       // should be unique across the cluster
+	string	        hostname        = 2;       // should be unique and work in DNS
 	int64	        cpus	        = 3;       // what's the point of int64 vs int32
 	int64	        memory	        = 4;       // in bytes
 	Current         current         = 5;       // what the state and values of the droplet is
-        DropletState                startState              = 6;  // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
-        string                      qemuMachine             = 7;  // the qemu machine type to use "pc-q35-9.0"
-        int64                       spicePort               = 8;  // preferred port to use for spice
+	DropletState    start_state     = 6;       // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
+	string	        qemu_machine    = 7;       // the qemu machine type to use "pc-q35-9.0"
+        int64           spice_port      = 8;       // preferred port to use for spice
 
-        string                      preferredHypervisor     = 9;  // the hypervisor to prefer to run the droplet on
-        string                      forceHypervisor         = 10; // use this hypervisor and this hypervisor only
-        string                      preferredArch           = 11; // the cpu arch to use "x86_64" (should really get this from the disk?)
+	string	 preferred_hypervisor   = 9;       // the hypervisor to prefer to run the droplet on
+	string	 force_hypervisor       = 10;      // use this hypervisor and this hypervisor only
+	string	 preferred_arch         = 11;      // the cpu arch to use "x86_64" (should really get this from the disk?)
 	repeated Network networks       = 12;      // really just mac addresses. should be unique across cluster
 	repeated Disk disks             = 13;      // disks to attach
 
-        string                      localOnly               = 14; // this is only defined locally on the hypervisor
-        string                      customXml               = 15; // if needed,
+	string          local_only      = 14;      // this is only defined locally on the hypervisor
+	string          custom_xml      = 15;      // if needed,
 	Archive         archive         = 16;      // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
 
         google.protobuf.Timestamp       unstable       = 39;    // the last time we heard anything from this droplet
-        google.protobuf.Duration    unstableTimeout         = 40; // the last time we heard anything from this droplet
+        google.protobuf.Duration        unstable_timeout       = 40;    // the last time we heard anything from this droplet
 }
 
 // volatile data. the current settings and values of things.
@@ -42,12 +40,12 @@ message Droplet {                                                 // `autogenpb:
 message Current {
         DropletState                    state           = 1;    // used to track the current state before taking any action
         string                          hypervisor      = 2;    // the current hypervisor the droplet is running on
-        int64                       startAttempts           = 3;  // how many times a start has been attempted
-        string                      fullXml                 = 4;  // the full libvirt xml to import
-        google.protobuf.Timestamp   lastPoll                = 5;  // the last time we heard anything from this droplet
-        string                      imageUrl                = 6;  // url to the image
-        google.protobuf.Timestamp   offSince                = 7;  // when the droplet was turned off
-        google.protobuf.Timestamp   onSince                 = 8;  // when the droplet was turned on
+        int64                           start_attempts  = 3;    // how many times a start has been attempted
+	string                          full_xml        = 4;    // the full libvirt xml to import
+        google.protobuf.Timestamp       last_poll       = 5;    // the last time we heard anything from this droplet
+        string                          image_url       = 6;    // url to the image
+        google.protobuf.Timestamp       off_since       = 7;    // when the droplet was turned off
+        google.protobuf.Timestamp       on_since        = 8;    // when the droplet was turned on
 }
 
 message Archive {
@@ -55,6 +53,7 @@ message Archive {
         google.protobuf.Timestamp       when            = 2;    // when it was archived
 }
 
+// virtual machine state
 enum DropletState {
 	ON              = 0;
 	OFF             = 1;
@@ -78,5 +77,5 @@ message Disk {
 	string	filename        = 1;
 	string	filepath        = 2;
 	int64	size    	= 3;
-        string                      qemuArch                = 4;  // what arch. example: "x86_64" or "riscv64"
+	string	qemu_arch       = 4;       // what arch. example: "x86_64" or "riscv64"
 }

event.proto

@@ -1,16 +1,13 @@
 syntax = "proto3";
-package virtpb;
+package virtbuf;
 
 import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
 import "google/protobuf/any.proto"; // Import the well-known type for Timestamp
-import "droplet.proto";
 
-// global settings for autogenpb `autogenpb:no-sort` `autogenpb:mutex`
-
-message Events {                                             // `autogenpb:marshal` `autogenpb:gui`
-        string                      uuid               = 1;  // `autogenpb:uuid:1e3a50c7-5916-4423-b33c-f0b977a7e446`
-        string                      version            = 2;  // `autogenpb:version:v0.0.1`
-        int64                       eventSize          = 3;  // max events to store in a single
+message Events {
+	string  uuid	        = 1;    // I guess why not just have this on each file
+	string  version         = 2;    // maybe can be used for protobuf schema change violations
+	int64   event_size      = 3;    // max events to store in a single
 	repeated Event events   = 4;    // all the events
 }
 
@@ -19,29 +16,22 @@ message Events {                                             // `autogenpb:marsh
 // at least for now in the early days. but maybe forever.
 // homelab clouds normally don't have many events.
 // we are talking less than 1 a minute. even 1 an hour is often a lot
-message Event {                                              // `autogenpb:marshal`
-        enum status {
-                DONE    = 0;
-                FAIL    = 1;
-                RUNNING = 2;
-        }
-        int32                       id                 = 1;  // `autogenpb:unique` // should be unique across the cluster
+message Event {
+	int32	        id     	        = 1;
 	EventType       etype           = 2;
-        string                      dropletName        = 3;  // name of the droplet
-        string                      dropletUuid        = 4;  // uuid of the droplet
+	string	        droplet         = 3;	// name of the droplet
+	string	        droplet_uuid    = 4;	// uuid of the droplet
 	string	        hypervisor	= 5;	// name of the hypervisor
-        string                      hypervisorUuid     = 6;  // uuid of the hypervisor
+	string	        hypervisor_uuid	= 6;	// uuid of the hypervisor
         google.protobuf.Timestamp start = 7;    // start time
         google.protobuf.Timestamp end   = 8;    // end time
-        string                      fieldName          = 9;  // the field name that changed
-        string                      origVal            = 10; // original value
-        string                      newVal             = 11; // new value
-        google.protobuf.Any         origAny            = 12; // anypb format. probably overkill
-        google.protobuf.Any         newAny             = 13; // anypb format
-        string                      error              = 14; // what went wrong
-        status                      state              = 15; // state of the event
-        Droplet                     droplet            = 16; // droplet
+        string          field_name      = 9;    // the field name that changed
+        string          orig_val        = 10;   // original value
+        string          new_val         = 11;   // new value
+        google.protobuf.Any orig_any    = 12;   // anypb format. probably overkill
+        google.protobuf.Any new_any     = 13;   // anypb format
 }
+
 enum EventType {
 	ADD	= 0;
 	DELETE = 1;
@@ -56,5 +46,4 @@ enum EventType {
 	FAIL	= 10;	// everything failed
 	CRASH   = 11;   // droplet hard crashed
 	CHANGE  = 12;   // droplet or hypervisor config change
-        EDIT    = 13;   // edit droplet settings
 }

example/Makefile

@@ -0,0 +1,15 @@
+build:
+	GO111MODULE=off go build
+	./example
+
+goimports:
+	goimports -w *.go
+
+prep:
+	go get -v -t -u
+
+run:
+	go run *.go
+
+clean:
+	-rm -f example

example/main.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	pb "go.wit.com/lib/protobuf/virtbuf"
+)
+
+//
+// saves entries in a config file
+//
+
+func main() {
+	TestWriteCluster()
+
+	var c *pb.NewCluster
+	c = pb.InitCluster()
+
+	// log.Println(aCluster.String())
+	// show the droplets to STDOUT
+	loop := c.DropletsAll() // get the list of droplets
+	for loop.Scan() {
+		d := loop.Droplet()
+		fmt.Println("\tdroplet =", d.Hostname, "preffered host:", d.PreferredHypervisor)
+	}
+
+	/*
+		// show the hypervisors to STDOUT
+		for _, h := range aCluster.Hypervisors {
+			fmt.Println("\thypervisor =", h.Hostname, h.GetMemoryPrintable())
+		}
+	*/
+
+	/*
+		json := aCluster.FormatJSON()
+		fmt.Println(json)
+
+		data, _ := aCluster.MarshalJSON()
+		fmt.Println(string(data))
+
+		text := aCluster.FormatTEXT()
+		fmt.Println(text)
+	*/
+}
+
+/*
+func marshalWriteToFile(myWriter *bufio.Writer, c *pb.NewCluster) {
+	buf, err := proto.Marshal(c)
+	if err != nil {
+		log.Fatal("marshaling error: ", err)
+	}
+	tmp, err := myWriter.Write(buf)
+	myWriter.Flush()
+	log.Println("bufio.Write() tmp, err = ", tmp, err)
+
+	buf, err = proto.Marshal(c)
+	tmp2, err := myWriter.Write(buf)
+	myWriter.Flush()
+	log.Println("bufio.Write() tmp2, err = ", tmp2, err)
+}
+*/
+
+func TestWriteCluster() {
+	c := pb.CreateSampleCluster(7)
+	os.Setenv("VIRTIGO_HOME", "/tmp/virtigo/")
+
+	if err := c.ConfigSave(); err != nil {
+		fmt.Println("configsave error", err)
+		os.Exit(-1)
+	}
+	// marshalUnmarshal()
+}
+
+/*
+func marshalUnmarshal() {
+	test := pb.CreateSampleCluster(7)
+	data, err := proto.Marshal(test)
+	if err != nil {
+		log.Fatal("marshaling error: ", err)
+	}
+
+	newTest := &pb.Cluster{}
+	err = proto.Unmarshal(data, newTest)
+	if err != nil {
+		log.Fatal("unmarshaling error: ", err)
+	} else {
+		log.Println("proto.Marshal() and proto.Unmarshal() worked")
+	}
+}
+*/

experiments.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+package virtbuf;
+
+import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
+import "google/protobuf/any.proto"; // Import the well-known type for Timestamp
+
+message WhatsThis {
+        // is it possible to have custom formatting in JSON and TEXT marshal/unmarshal ?
+        WhatInfo humantest   = 1;
+
+        google.protobuf.Timestamp end   = 2;    // end time
+        google.protobuf.Any orig_val    = 3;    // original value
+        google.protobuf.Any new_val     = 4;   // new value
+}
+
+// this is for exerimenting
+message WhatInfo {
+        int64 capacity = 1;  // Stores the storage capacity in bytes.
+}

helpers.go

@@ -1,17 +1,81 @@
-package virtpb
+package virtbuf
 
 // functions to import and export the protobuf
 // data to and from config files
 
-func InitCluster() *OldCluster {
-	var c *OldCluster
-	c = new(OldCluster)
+import (
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/encoding/prototext"
+	"google.golang.org/protobuf/proto"
+)
+
+func InitCluster() *NewCluster {
+	var c *NewCluster
+	c = new(NewCluster)
 	c.d = new(Droplets)
 	c.H = new(Hypervisors)
 	c.e = new(Events)
 	return c
 }
 
-func (c *OldCluster) DropletsAll() *DropletScanner {
-	return c.d.All()
+// human readable JSON
+func (d *Droplets) FormatJSON() string {
+	return protojson.Format(d)
+}
+
+func (d *Droplet) FormatJSON() string {
+	return protojson.Format(d)
+}
+
+func (e *Events) FormatJSON() string {
+	return protojson.Format(e)
+}
+
+func (h *Hypervisors) FormatJSON() string {
+	return protojson.Format(h)
+}
+
+// apparently this isn't supposed to be used?
+// https://protobuf.dev/reference/go/faq/#unstable-text
+// this is a shame because this is much nicer output than JSON Format()
+func (d *Droplets) FormatTEXT() string {
+	return prototext.Format(d)
+}
+
+func (d *Droplet) FormatTEXT() string {
+	return prototext.Format(d)
+}
+
+func (e *Events) FormatTEXT() string {
+	return prototext.Format(e)
+}
+
+// marshal
+func (d *Droplets) MarshalJSON() ([]byte, error) {
+	return protojson.Marshal(d)
+}
+
+func (d *Droplet) MarshalJSON() ([]byte, error) {
+	return protojson.Marshal(d)
+}
+
+func (e *Events) MarshalJSON() ([]byte, error) {
+	return protojson.Marshal(e)
+}
+
+// unmarshal
+func (d *Droplets) UnmarshalJSON(data []byte) error {
+	return protojson.Unmarshal(data, d)
+}
+
+func (d *Droplet) UnmarshalJSON(data []byte) error {
+	return protojson.Unmarshal(data, d)
+}
+
+func (e *Events) UnmarshalJSON(data []byte) error {
+	return protojson.Unmarshal(data, e)
+}
+
+func (d *Droplet) Unmarshal(data []byte) error {
+	return proto.Unmarshal(data, d)
 }

human.go

@@ -1,9 +1,9 @@
-package virtpb
+package virtbuf
 
 // mostly just functions related to making STDOUT
 // more readable by us humans
 
-// also function shortcuts the do fixed limited formatting (it's like COBOL)
+// also function shortcuts the do limited formatting (haha, who remembers COBOL?)
 // so reporting tables of the status of what droplets and hypervisors
 // are in text columns and rows that can be easily read in a terminal
 
@@ -14,8 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"google.golang.org/protobuf/types/known/timestamppb"
-
 	"go.wit.com/log"
 )
 
@@ -97,7 +95,7 @@ func FormatDuration(d time.Duration) string {
 	ms := int(d.Milliseconds())
 	if ms > 100 {
 		// todo: print .3s, etc ?
-		return fmt.Sprintf("%1.2fs", float64(seconds)/1000)
+		return fmt.Sprintf("%1.2fs", seconds/1000)
 	}
 	if ms > 0 {
 		result += fmt.Sprintf("%dms", ms)
@@ -148,26 +146,34 @@ func (d *Droplet) SprintDumpHeader() string {
 		macs = append(macs, n.Mac)
 	}
 
+	// this line in golang could replace 80 lines of COBOL
 	header := fmt.Sprintf("%-4.4s%20s %-8s", d.Current.State, strings.Join(macs, " "), d.Current.Hypervisor)
 
 	if d.Current == nil {
-		return header
+		d.Current = new(Current)
 	}
 
-	if d.Current.OnSince == nil {
-		d.Current.OnSince = timestamppb.New(time.Now())
-	}
-
-	t := time.Since(d.Current.OnSince.AsTime()) // time since 'ON'
-	dur := FormatDuration(t)
-
 	switch d.Current.State {
 	case DropletState_ON:
+		var dur string
+		if d.Current.OnSince != nil {
+			dur = ""
+		} else {
+			t := time.Since(d.Current.OnSince.AsTime()) // time since 'ON'
+			dur = FormatDuration(t)
+		}
 		header += fmt.Sprintf(" (on :%3s)", dur)
 	case DropletState_OFF:
+		var dur string
+		if d.Current.OffSince != nil {
+			dur = ""
+		} else {
+			t := time.Since(d.Current.OffSince.AsTime()) // time since 'OFF'
+			dur = FormatDuration(t)
+		}
 		header += fmt.Sprintf(" (off:%3s)", dur)
 	default:
-		header += fmt.Sprintf(" (?? :%3s)", dur)
+		header += fmt.Sprintf(" (?? :%3s)", "")
 	}
 	return header
 }
@@ -185,7 +191,7 @@ func (d *Droplet) DumpDroplet(w http.ResponseWriter, r *http.Request) (string, e
 	return t, nil
 }
 
-func (c *OldCluster) DumpDroplet(w http.ResponseWriter, r *http.Request) (string, error) {
+func (c *NewCluster) DumpDroplet(w http.ResponseWriter, r *http.Request) (string, error) {
 	hostname := r.URL.Query().Get("hostname")
 	d := c.FindDropletByName(hostname)
 	if d == nil {

hypervisor.proto

@@ -1,27 +1,24 @@
 syntax = "proto3";
-package virtpb;
+package virtbuf;
 
-import "google/protobuf/timestamp.proto";
-
-message Hypervisors {                                     // `autogenpb:marshal` `autogenpb:gui`
-        string                      uuid            = 1;  // `autogenpb:uuid:6e3aa8b9-cf98-40f6-af58-3c6ad1edf4d4`
-        string                      version         = 2;  // `autogenpb:version:v0.0.1`
+message Hypervisors {
+	string  uuid	        = 1;    // I guess why not just have this on each file
+	string  version         = 2;    // maybe can be used for protobuf schema change violations
 	repeated Hypervisor hypervisors = 3;
 }
 
 message Hypervisor {
-        string                      uuid            = 1;  // `autogenpb:unique`
-        string                      hostname        = 2;  // `autogenpb:unique`
+	string  	uuid            = 1;
+	string  	hostname        = 2;
 	bool    	active          = 3; // is allowed to start new droplets
 	int64   	cpus	        = 4;
 	int64   	memory	        = 5; // in bytes
 	string	        comment	        = 6;
 	bool            autoscan        = 7; // to scan or not to scan by virtigo
         HypervisorArch  arch            = 8;
-        int64                       killcount       = 9;  // in bytes
-        google.protobuf.Timestamp   lastPoll        = 10; // the last time we heard anything
 }
 
+// think about this more
 enum HypervisorArch {
 	RISCV64   = 0;
 	X86_64    = 1;

newCluster.go

@@ -0,0 +1,19 @@
+package virtbuf
+
+import (
+	sync "sync"
+
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+type NewCluster struct {
+	sync.RWMutex
+
+	Dirs            []string
+	d               *Droplets
+	H               *Hypervisors
+	e               *Events
+	Unstable        *timestamppb.Timestamp
+	UnstableTimeout *durationpb.Duration
+}

oldCluster.go

@@ -1,48 +0,0 @@
-package virtpb
-
-import (
-	sync "sync"
-
-	durationpb "google.golang.org/protobuf/types/known/durationpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
-)
-
-type OldCluster struct {
-	sync.RWMutex
-
-	Dirs            []string
-	d               *Droplets
-	H               *Hypervisors
-	e               *Events
-	Unstable        *timestamppb.Timestamp
-	UnstableTimeout *durationpb.Duration
-}
-
-func (c *OldCluster) GetDropletsPB() *Droplets {
-	return c.d
-}
-
-func (c *OldCluster) GetHypervisorsPB() *Hypervisors {
-	return c.H
-}
-
-func (c *OldCluster) GetEventsPB() *Events {
-	return c.e
-}
-
-// adds a new droplet. enforce unique hostnames
-func (c *OldCluster) AddDroplet(newd *Droplet) bool {
-	c.Lock()
-	defer c.Unlock()
-
-	for _, d := range c.d.Droplets {
-		if newd.Hostname == d.Hostname {
-			// boo. that one is already here
-			return false
-		}
-	}
-
-	// everything is ok, this hostname is new
-	c.d.Droplets = append(c.d.Droplets, newd)
-	return true
-}

sampleData.go

@@ -1,4 +1,4 @@
-package virtpb
+package virtbuf
 
 import (
 	"fmt"
@@ -39,6 +39,22 @@ func CreateSampleHypervisor(hostname string, mem int) *Hypervisor {
 	return h
 }
 
+func CreateExperiment(total int) *WhatsThis {
+	var e *WhatsThis
+	e = new(WhatsThis)
+
+	//	info := StorageInfo{Capacity: 64}
+	//	e.Humantest = &info
+	if e.Humantest == nil {
+		var newInfo WhatInfo
+		newInfo = WhatInfo{Capacity: 64}
+		e.Humantest = &newInfo
+	} else {
+		e.Humantest.Capacity = SetGB(total * 32)
+	}
+	return e
+}
+
 func CreateSampleEvents(total int) *Events {
 	var e *Events
 	e = new(Events)
@@ -51,7 +67,7 @@ func CreateSampleEvents(total int) *Events {
 	return e
 }
 
-func CreateSampleCluster(total int) *OldCluster {
+func CreateSampleCluster(total int) *NewCluster {
 	c := InitCluster()
 
 	for i := 0; i < total; i++ {

scanIterator.go

@@ -0,0 +1,106 @@
+package virtbuf
+
+import (
+	"fmt"
+	"os"
+
+	"go.wit.com/log"
+)
+
+type DropletIterator struct {
+	droplets []*Droplet
+	index    int
+}
+
+// NewDropletIterator initializes a new iterator.
+func NewDropletIterator(droplets []*Droplet) *DropletIterator {
+	return &DropletIterator{droplets: droplets}
+}
+
+// Scan moves to the next element and returns false if there are no more droplets.
+func (it *DropletIterator) Scan() bool {
+	if it.index >= len(it.droplets) {
+		return false
+	}
+	it.index++
+	return true
+}
+
+// Droplet returns the current droplet.
+func (it *DropletIterator) Droplet() *Droplet {
+	if it.droplets[it.index-1] == nil {
+		for i, d := range it.droplets {
+			fmt.Println("i =", i, d)
+		}
+		fmt.Println("len =", len(it.droplets))
+		fmt.Println("droplet == nil", it.index, it.index-1)
+		os.Exit(-1)
+	}
+	return it.droplets[it.index-1]
+}
+
+// Use Scan() in a loop, similar to a while loop
+//
+//	for iterator.Scan() {
+//		d := iterator.Droplet()
+//		fmt.Println("Droplet UUID:", d.Uuid)
+//	}
+func (c *NewCluster) GetDropletIterator() *DropletIterator {
+	dropletPointers := c.SelectDropletPointers()
+
+	iterator := NewDropletIterator(dropletPointers)
+
+	return iterator
+}
+
+func (c *NewCluster) DropletsAll() *DropletIterator {
+	dropletPointers := c.SelectDropletAll()
+
+	iterator := NewDropletIterator(dropletPointers)
+
+	return iterator
+}
+
+// SelectDropletPointers safely returns a slice of pointers to Droplet records.
+func (c *NewCluster) SelectDropletAll() []*Droplet {
+	c.RLock()
+	defer c.RUnlock()
+
+	// Create a new slice to hold pointers to each Droplet
+	// dropletPointers := make([]*Droplet, len(c.E.Droplets))
+	var dropletPointers []*Droplet
+	if c.d == nil {
+		log.Info("SelectDropletsAll() c.d == nil")
+		// os.Exit(-1)
+	}
+	for _, d := range c.d.Droplets {
+		if d == nil {
+			continue
+		}
+		if d.Archive != nil {
+			continue
+		}
+		dropletPointers = append(dropletPointers, d) // Copy pointers for safe iteration
+	}
+
+	return dropletPointers
+}
+
+// SelectDropletPointers safely returns a slice of pointers to Droplet records.
+func (c *NewCluster) SelectDropletPointers() []*Droplet {
+	c.RLock()
+	defer c.RUnlock()
+
+	// Create a new slice to hold pointers to each Droplet
+	// dropletPointers := make([]*Droplet, len(c.E.Droplets))
+	dropletPointers := make([]*Droplet, 1)
+	if c.d == nil {
+		log.Info("c.d == nil")
+		os.Exit(-1)
+	}
+	for _, d := range c.d.Droplets {
+		dropletPointers = append(dropletPointers, d) // Copy pointers for safe iteration
+	}
+
+	return dropletPointers
+}

storageinfo.go

@@ -1,6 +1,11 @@
-package virtpb
+package virtbuf
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+)
 
-/*
 // MarshalJSON custom marshals the WhatInfo struct to JSON
 func (s WhatInfo) MarshalJSON() ([]byte, error) {
 	capacityStr := fmt.Sprintf("%d GB", s.Capacity)
@@ -32,6 +37,7 @@ func (s *WhatInfo) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+/*
 func main() {
 	info := WhatInfo{Capacity: 64}
 

work/google.golang.org/protobuf

@@ -0,0 +1 @@
+Subproject commit 29947bbd165cd2034df6d15ae06feb83af40c572