Compare commits

..

37 Commits

Author SHA1 Message Date
Jeff Carr e6fb7352ae quiet debugging 2025-04-23 02:43:18 -05:00
Jeff Carr c9ef4f0b82 make an EDIT event 2025-04-22 20:50:00 -05:00
Jeff Carr 688b5039a0 new ConfigSave() 2025-04-21 20:54:45 -05:00
Jeff Carr 814d36b9c9 working on droplet start event 2025-04-21 20:54:43 -05:00
Jeff Carr a07033d181 move these into cluster.pb 2025-04-21 20:54:38 -05:00
Jeff Carr 7cdb2a33ef do a whole cluster protobuf at once 2025-04-20 19:41:02 -05:00
Jeff Carr 63148556af add Name and allow multiple URLs 2025-04-12 11:28:05 -05:00
Jeff Carr a510dd6474 this is actually a 'scanner' 2025-03-27 07:31:50 -05:00
Jeff Carr e4345c8ad6 moving to a cluster.proto config file 2025-03-24 21:54:13 -05:00
Jeff Carr 276c5cec2f pass out hypervisors 2025-03-11 04:02:45 -05:00
Jeff Carr e78fc1235e rm code now made by autogenpb 2025-03-10 18:10:40 -05:00
Jeff Carr c82997ed61 hypervisor poll time attempt 1 2025-02-23 13:13:30 -06:00
Jeff Carr 4aef241137 update worked to gocui 2025-02-23 13:13:30 -06:00
Jeff Carr c6cb62c86d sends the table across 2025-02-23 13:13:30 -06:00
Jeff Carr 0f546d57fc set uuid 2025-02-23 13:13:30 -06:00
Jeff Carr 13159b5b64 add daemon killcount and lastpoll 2025-02-23 13:13:30 -06:00
Jeff Carr 9dfcbb0432 early attempt at pb table update() 2025-02-23 13:13:30 -06:00
Jeff Carr 3ca7403aa6 minor 2025-02-23 13:13:30 -06:00
Jeff Carr 65f9089a7a remove _ names rather than fix the autogenpb parser 2025-02-22 18:17:31 -06:00
Jeff Carr b25d86f277 also need gui table support for this 2025-02-22 18:14:49 -06:00
Jeff Carr d40dc91130 switch virtbuf to virtpb 2025-02-22 17:45:59 -06:00
Jeff Carr 2381c65887 cleanups for gui pb tables 2025-02-22 15:23:40 -06:00
Jeff Carr ddc0410126 fix proto files to conform with autogenpb 2025-02-07 04:41:25 -06:00
Jeff Carr 9160268326 duh. TRUNCATE on new files 2024-12-11 13:54:36 -06:00
Jeff Carr 0888e51c91 trick to easily detect protobuf libraries 2024-12-02 05:15:58 -06:00
Jeff Carr 8f3953159a autogenpb now configured in .proto files 2024-12-01 22:24:01 -06:00
Jeff Carr aeac6b5af7 minor autogenpb fixes 2024-12-01 18:59:34 -06:00
Jeff Carr afd0bd6428 switched to autogenpb 2024-12-01 18:41:00 -06:00
Jeff Carr 5b883de7b9 never put binaries in go libraries 2024-12-01 17:57:35 -06:00
Jeff Carr 3046ff335f stuff
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-18 05:32:37 -06:00
Jeff Carr 81cbb6e9d7 things for create. might be duplicates
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-16 05:22:11 -06:00
Jeff Carr 74da63276e rename
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-16 01:52:43 -06:00
Jeff Carr 3b63a3af24 better formatting
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-13 15:29:37 -06:00
Jeff Carr 907981a92d rename Cluster
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-07 05:04:11 -06:00
Jeff Carr b9766ce266 minor
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-06 19:23:56 -06:00
Jeff Carr 4fdb1934ff not much here yet
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-06 19:14:14 -06:00
Jeff Carr 2f715b47d5 attempt at instructions
Signed-off-by: Jeff Carr <jcarr@wit.com>
2024-11-02 23:56:56 -05:00
23 changed files with 357 additions and 586 deletions

4
.gitignore vendored
View File

@ -1,5 +1,3 @@
go.*
*.pb.go
example/example
*.swp

3
.protobuf Normal file
View File

@ -0,0 +1,3 @@
droplet.proto
event.proto
hypervisor.proto

View File

@ -1,23 +1,17 @@
# You must use the current protoc-gen-go
# You must use the current google protoc-gen-go
#
# go-clone --go-src google.golang.org/protobuf
# cd ~/go/src/google.golang.org/protobuf/cmd/protoc-gen-go
# go install
all: proto goimports vet
all: droplet.pb.go hypervisor.pb.go event.pb.go experiments.pb.go
make -C example
vet: lint
GO111MODULE=off go vet
lint:
-buf lint droplet.proto
vet:
@GO111MODULE=off go vet
@echo this go library package builds okay
# autofixes your import headers in your golang files
goimports:
goimports -w *.go
make -C example goimports
redomod:
rm -f go.*
@ -27,32 +21,20 @@ redomod:
clean:
rm -f *.pb.go
-rm -f go.*
make -C example clean
proto:droplet.pb.go hypervisor.pb.go event.pb.go cluster.pb.go
droplet.pb.go: droplet.proto
# protoc --go_out=. droplet.proto
# This is switched over to use the new protoc-gen-go from google.golang.org/protobuf/cmd/protoc-gen-go
# the debian one (2024/10/21) seems to be the older/original one from github.com/golang/protobuf/protoc-gen-go
cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/virtbuf \
--go_opt=Mdroplet.proto=go.wit.com/lib/protobuf/virtbuf \
droplet.proto
autogenpb --proto droplet.proto
hypervisor.pb.go: hypervisor.proto
cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/virtbuf \
--go_opt=Mhypervisor.proto=go.wit.com/lib/protobuf/virtbuf \
hypervisor.proto
autogenpb --proto hypervisor.proto
event.pb.go: event.proto
cd ~/go/src && protoc --go_out=. \
--proto_path=go.wit.com/lib/protobuf/virtbuf \
--go_opt=Mevent.proto=go.wit.com/lib/protobuf/virtbuf \
event.proto
autogenpb --proto event.proto
experiments.pb.go: experiments.proto
cd ~/go/src && protoc --go_out=. \
--proto_path=go.wit.com/lib/protobuf/virtbuf \
--go_opt=Mexperiments.proto=go.wit.com/lib/protobuf/virtbuf \
experiments.proto
cluster.pb.go: cluster.proto
autogenpb --proto cluster.proto
deps:
apt install golang-goprotobuf-dev

View File

@ -1,13 +1,6 @@
They are what to use for starting droplets with virsh,
but for making a cluster or "home cloud" with virtigo
This go library handles the protobuf files
and various functions for virtigo.
When possible & sensible, use the same variable names as libvirt
virtigo needs to coordinate all the virtual machine
information centrally. The information in the libvirt xml
files are imported into these protobuf definitions and
passed around as protobufs
virtigo recreates the libvirt xml files and calls
'virsh create'
You must build the protobuf files using autogenpb
go install go.wit.com/apps/autogenpb@latest

62
add.go
View File

@ -1,7 +1,6 @@
package virtbuf
package virtpb
import (
"errors"
"fmt"
"time"
@ -9,7 +8,8 @@ import (
"go.wit.com/log"
)
func (c *NewCluster) InitDroplet(hostname string) (*Droplet, error) {
/*
func (c *OldCluster) InitDroplet(hostname string) (*Droplet, error) {
var d *Droplet
d = new(Droplet)
d.Current = new(Current)
@ -28,12 +28,13 @@ func (c *NewCluster) InitDroplet(hostname string) (*Droplet, error) {
return d, nil
}
func (c *NewCluster) appendDroplet(d *Droplet) {
func (c *Cluster) appendDroplet(d *Droplet) {
c.Lock()
defer c.Unlock()
c.d.Droplets = append(c.d.Droplets, d)
}
*/
// can the json protobuf output use a string and have a type handler
// to convert it back to int64?
@ -49,10 +50,10 @@ func (x *Hypervisor) SetMemoryGB(gb int) {
x.Memory = int64(gb * 1024 * 1024 * 1024)
}
func (c *NewCluster) FindDropletByName(name string) *Droplet {
loop := c.DropletsAll() // get the list of droplets
func (c *OldCluster) FindDropletByName(name string) *Droplet {
loop := c.d.All() // get the list of droplets
for loop.Scan() {
d := loop.Droplet()
d := loop.Next()
if d.Hostname == name {
return d
}
@ -60,7 +61,20 @@ func (c *NewCluster) FindDropletByName(name string) *Droplet {
return nil
}
func (c *NewCluster) FindHypervisorByName(name string) *Hypervisor {
func (c *OldCluster) FindDropletByUuid(id string) *Droplet {
/*
log.Info("START FIND", id)
loop := c.d.All() // get the list of droplets
for loop.Scan() {
d := loop.Next()
log.Info("droplet:", d.Hostname, d.Uuid)
}
log.Info("END FIND", id)
*/
return c.d.FindByUuid(id)
}
func (c *OldCluster) FindHypervisorByName(name string) *Hypervisor {
for _, h := range c.H.Hypervisors {
if h.Hostname == name {
return h
@ -69,7 +83,7 @@ func (c *NewCluster) FindHypervisorByName(name string) *Hypervisor {
return nil
}
func (c *NewCluster) AddHypervisor(hostname string, cpus int, mem int) *Hypervisor {
func (c *OldCluster) AddHypervisor(hostname string, cpus int, mem int) *Hypervisor {
h := c.FindHypervisorByName(hostname)
if h != nil {
return h
@ -90,15 +104,25 @@ func (c *NewCluster) AddHypervisor(hostname string, cpus int, mem int) *Hypervis
return h
}
func (c *NewCluster) AddEvent(e *Event) {
func (c *OldCluster) AddEvent(e *Event) {
c.e.Events = append(c.e.Events, e)
}
func (c *NewCluster) AddDroplet(d *Droplet) {
c.d.Droplets = append(c.d.Droplets, d)
// creates a new droplet with default values
func NewDefaultDroplet(hostname string) *Droplet {
id := uuid.New() // Generate a new UUID
d := &Droplet{
Uuid: id.String(),
Hostname: hostname,
Cpus: int64(2),
}
d.Memory = SetGB(2)
d.StartState = DropletState_OFF
return d
}
func (c *NewCluster) AddDropletSimple(uuid string, hostname string, cpus int, mem int) *Droplet {
func (c *OldCluster) AddDropletSimple(uuid string, hostname string, cpus int, mem int) *Droplet {
d := c.FindDropletByName(hostname)
if d != nil {
return d
@ -120,7 +144,7 @@ func (c *NewCluster) AddDropletSimple(uuid string, hostname string, cpus int, me
}
// This isn't for the marketing department
func (c *NewCluster) AddDropletLocal(name string, hypername string) *Droplet {
func (c *OldCluster) AddDropletLocal(name string, hypername string) *Droplet {
d := &Droplet{
Hostname: name,
}
@ -138,10 +162,10 @@ func (c *NewCluster) AddDropletLocal(name string, hypername string) *Droplet {
return d
}
func (c *NewCluster) BlankFields() {
loop := c.DropletsAll() // get the list of droplets
func (c *OldCluster) BlankFields() {
loop := c.d.All() // get the list of droplets
for loop.Scan() {
d := loop.Droplet()
d := loop.Next()
d.Current = nil
}
}
@ -150,7 +174,7 @@ func (epb *Events) AppendEvent(e *Event) {
epb.Events = append(epb.Events, e)
}
func (c *NewCluster) ClusterStable() (bool, string) {
func (c *OldCluster) ClusterStable() (bool, string) {
last := time.Since(c.Unstable.AsTime())
if last > c.UnstableTimeout.AsDuration() {
// the cluster has not been stable for 133 seconds
@ -162,7 +186,7 @@ func (c *NewCluster) ClusterStable() (bool, string) {
}
// check the cluster and droplet to make sure it's ready to start
func (c *NewCluster) DropletReady(d *Droplet) (bool, string) {
func (c *OldCluster) DropletReady(d *Droplet) (bool, string) {
if c == nil {
return false, "cluster == nil"
}

View File

@ -1,4 +1,4 @@
package virtbuf
package virtpb
// thank chatgpt for this because why. why write this if you can have it
// kick this out in 30 seconds
@ -14,7 +14,7 @@ func backupDir(srcDir string, destDir string) error {
// Create the destination directory
err := os.MkdirAll(destDir, os.ModePerm)
if err != nil {
log.Println("Failed to create directory: %v", err)
log.Printf("Failed to create directory: %v\n", err)
return err
}
@ -41,7 +41,7 @@ func backupDir(srcDir string, destDir string) error {
})
if err != nil {
log.Println("Failed to copy files: %v", err)
log.Printf("Failed to copy files: %v\n", err)
return err
}
return nil

View File

@ -1,4 +1,4 @@
package virtbuf
package virtpb
import (
// "reflect"
@ -34,7 +34,6 @@ func convertToAnypb(x any) *anypb.Any {
return a
default:
log.Error(errors.New("convertToAnypb() unknown type"), "v =", v, "x =", x)
return nil
}
return nil
}
@ -65,7 +64,6 @@ func convertToString(x any) string {
default:
log.Info("convertToSTring() unknown type", v)
log.Error(errors.New("convertToSTring() unknown type"), "v =", v, "x =", x)
return ""
}
return ""
}
@ -75,7 +73,7 @@ func (d *Droplet) NewChangeEvent(fname string, origval any, newval any) *Event {
var e *Event
e = new(Event)
e.Droplet = d.Hostname
e.DropletName = d.Hostname
e.OrigVal = convertToString(origval)
e.NewVal = convertToString(newval)
e.FieldName = fname
@ -101,12 +99,12 @@ func NewAddEvent(a any, fname string, newval any) *Event {
case *Droplet:
var d *Droplet
d = a.(*Droplet)
e.Droplet = d.Hostname
e.DropletName = d.Hostname
case nil:
e.Droplet = "<nil>"
e.DropletName = "<nil>"
default:
log.Info("newAddEvent() unknown type", v)
e.Droplet = "on something somewhere"
e.DropletName = "on something somewhere"
}
e.NewVal = convertToString(newval)
@ -165,7 +163,7 @@ func (d *Droplet) SetState(newState DropletState) {
}
// records an event that the droplet changed state (aka turned on, turned off, etc)
func (c *NewCluster) ChangeDropletState(d *Droplet, newState DropletState) error {
func (c *OldCluster) ChangeDropletState(d *Droplet, newState DropletState) error {
if c == nil {
return errors.New("cluster is nil")
}
@ -179,7 +177,7 @@ func (c *NewCluster) ChangeDropletState(d *Droplet, newState DropletState) error
var e *Event
e = new(Event)
e.Droplet = d.Hostname
e.DropletName = d.Hostname
e.OrigVal = convertToString(d.Current.State)
e.NewVal = convertToString(newState)
e.FieldName = "status"
@ -192,7 +190,7 @@ func (c *NewCluster) ChangeDropletState(d *Droplet, newState DropletState) error
}
// records an event that the droplet migrated to another hypervisor
func (c *NewCluster) DropletMoved(d *Droplet, newh *Hypervisor) error {
func (c *OldCluster) DropletMoved(d *Droplet, newh *Hypervisor) error {
if c == nil {
return errors.New("cluster is nil")
}
@ -211,7 +209,7 @@ func (c *NewCluster) DropletMoved(d *Droplet, newh *Hypervisor) error {
var e *Event
e = new(Event)
e.Droplet = d.Hostname
e.DropletName = d.Hostname
e.OrigVal = d.Current.Hypervisor
e.NewVal = newh.Hostname
e.FieldName = "droplet migrate"

23
cluster.proto Normal file
View File

@ -0,0 +1,23 @@
syntax = "proto3";
package virtpb;
import "google/protobuf/timestamp.proto";
import "droplet.proto";
import "hypervisor.proto";
import "event.proto";
message Cluster { // `autogenpb:marshal`
string uuid = 1; // `autogenpb:unique`
string name = 2;
repeated string URL = 3;
google.protobuf.Timestamp ctime = 4; // when the cluster was created
Droplets droplets = 5;
Hypervisors hypervisors = 6;
Events events = 7;
}
message Clusters { // `autogenpb:marshal`
string uuid = 1; // `autogenpb:uuid:57ddd763-75f6-4003-bf0e-8dd0f8a44044`
string version = 2; // `autogenpb:version:v0.0.1`
repeated Cluster clusters = 3;
}

111
config.go
View File

@ -1,4 +1,4 @@
package virtbuf
package virtpb
// functions to import and export the protobuf
// data to and from config files
@ -9,14 +9,37 @@ import (
"os"
"path/filepath"
"go.wit.com/log"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/encoding/prototext"
"google.golang.org/protobuf/reflect/protoreflect"
)
func (c *Cluster) ConfigSave() error {
name := c.Name
if name == "" {
name = c.Uuid
}
fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), name+".pb")
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
defer cfgfile.Close()
if err != nil {
fmt.Println("open config file :", err)
return err
}
log.Info("ConfigSave()", fullname)
data, err := c.Marshal()
if err != nil {
fmt.Println("cluster Marshal() err:", err)
return err
}
fmt.Fprintln(cfgfile, data)
return nil
}
// writes out the cluster information it seperate files
// to make it humanly possible to hand edit things as needed
func (c *NewCluster) ConfigSave() error {
func (c *OldCluster) ConfigSave() error {
// try to backup the current cluster config files
if err := backupConfig(); err != nil {
return err
@ -25,31 +48,21 @@ func (c *NewCluster) ConfigSave() error {
// make a new droplets struct
var dcopy *Droplets
dcopy = new(Droplets)
loop := c.DropletsAll() // get the list of droplets
loop := c.d.All() // get the list of droplets
for loop.Scan() {
d := loop.Droplet()
var newd Droplet
newd = *d
dcopy.Droplets = append(dcopy.Droplets, &newd)
d := loop.Next()
dcopy.Droplets = append(dcopy.Droplets, d)
}
// delete all the Current data so it's not put in the config file
for _, drop := range dcopy.Droplets {
drop.Current = nil
}
if err := ConfigWriteJSON(dcopy, "droplets.json"); err != nil {
fmt.Println("droplets.json write failed")
return err
}
if err := ConfigWriteTEXT(dcopy, "droplets.text"); err != nil {
fmt.Println("droplets.json write failed")
return err
}
c.configWriteDroplets()
if err := ConfigWriteJSON(c.H, "hypervisors.json"); err != nil {
fmt.Println("hypervisors.json write failed")
return err
}
if err := ConfigWriteTEXT(c.H, "hypervisors.text"); err != nil {
fmt.Println("hypervisors.json write failed")
return err
@ -66,27 +79,27 @@ func (c *NewCluster) ConfigSave() error {
return nil
}
func (c *NewCluster) ConfigLoad() error {
func (c *OldCluster) ConfigLoad() error {
if c == nil {
return errors.New("It's not safe to run ConfigLoad() on a nil cluster")
}
if data, err := loadFile("droplets.json"); err == nil {
if err = protojson.Unmarshal(data, c.d); err != nil {
fmt.Println("broken droplets.json config file")
if data, err := loadFile("droplets.text"); err == nil {
if err = prototext.Unmarshal(data, c.d); err != nil {
fmt.Println("broken droplets.text config file")
return err
}
} else {
return err
}
if data, err := loadFile("hypervisors.json"); err == nil {
if err = protojson.Unmarshal(data, c.H); err != nil {
fmt.Println("broken hypervisors.json config file")
if data, err := loadFile("hypervisors.text"); err == nil {
if err = prototext.Unmarshal(data, c.H); err != nil {
fmt.Println("broken hypervisors.text config file")
return err
}
} else {
fmt.Println("ERROR HERE IN Hypervisors")
log.Warn("ERROR HERE IN Hypervisors")
return err
}
@ -95,8 +108,11 @@ func (c *NewCluster) ConfigLoad() error {
// does it not stay allocated after this function ends?
c.e = new(Events)
}
if err := c.e.loadEvents(); err != nil {
return err
// ignore events.pb since these should be sent elsewhere
log.Warn("Events failed to load, ignoring:", err)
return nil
}
return nil
}
@ -144,7 +160,7 @@ func loadFile(filename string) ([]byte, error) {
func ConfigWriteJSON(a any, filename string) error {
fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), filename)
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE, 0666)
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
defer cfgfile.Close()
if err != nil {
fmt.Println("open config file :", err)
@ -159,9 +175,26 @@ func ConfigWriteJSON(a any, filename string) error {
return nil
}
func (c *OldCluster) configWriteDroplets() error {
fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), "droplets.new.text")
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
defer cfgfile.Close()
if err != nil {
fmt.Println("open config file :", err)
return err
}
loop := c.d.All() // get the list of droplets
for loop.Scan() {
d := loop.Next()
text := prototext.Format(d)
fmt.Fprintln(cfgfile, text)
}
return nil
}
func ConfigWriteTEXT(a any, filename string) error {
fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), filename)
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE, 0666)
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
defer cfgfile.Close()
if err != nil {
fmt.Println("open config file :", err)
@ -176,22 +209,18 @@ func ConfigWriteTEXT(a any, filename string) error {
return nil
}
func (c *NewCluster) configWriteDroplets() error {
fullname := filepath.Join(os.Getenv("VIRTIGO_HOME"), "droplets.new.text")
cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE, 0666)
defer cfgfile.Close()
if err != nil {
fmt.Println("open config file :", err)
return err
func (c *Clusters) ConfigLoad() error {
if c == nil {
return errors.New("It's not safe to run ConfigLoad() on a nil cluster")
}
loop := c.DropletsAll() // get the list of droplets
for loop.Scan() {
d := loop.Droplet()
var newd Droplet
newd = *d
newd.Current = nil
text := prototext.Format(&newd)
fmt.Fprintln(cfgfile, text)
if data, err := loadFile("cluster.text"); err == nil {
if err = prototext.Unmarshal(data, c); err != nil {
fmt.Println("broken cluster.textconfig file")
return err
}
} else {
return err
}
return nil
}

View File

@ -1,4 +1,4 @@
package virtbuf
package virtpb
// thank chatgpt for this because why. why write this if you can have it
// kick this out in 30 seconds

View File

@ -1,81 +1,82 @@
syntax = "proto3";
package virtbuf;
package virtpb;
import "google/protobuf/duration.proto"; // Import the well-known type for Timestamp
import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
message Droplets {
string uuid = 1; // I guess why not just have this on each file
string version = 2; // maybe can be used for protobuf schema change violations
repeated Droplet droplets = 3;
// global settings for autogenpb `autogenpb:mutex`
message Droplets { // `autogenpb:marshal` `autogenpb:gui`
string uuid = 1; // `autogenpb:uuid:d5d492e2-38d4-476b-86f3-f5abf01f9d6d`
string version = 2; // `autogenpb:version:v0.0.1`
repeated Droplet droplets = 3;
}
message Droplet {
string uuid = 1; // should be unique across the cluster
string hostname = 2; // should be unique and work in DNS
int64 cpus = 3; // what's the point of int64 vs int32
int64 memory = 4; // in bytes
Current current = 5; // what the state and values of the droplet is
DropletState start_state = 6; // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
string qemu_machine = 7; // the qemu machine type to use "pc-q35-9.0"
int64 spice_port = 8; // preferred port to use for spice
message Droplet { // `autogenpb:marshal`
string uuid = 1; // `autogenpb:unique` // should be unique across the cluster
string hostname = 2; // `autogenpb:unique` // should be unique and work in DNS
int64 cpus = 3; // what's the point of int64 vs int32
int64 memory = 4; // in bytes
Current current = 5; // what the state and values of the droplet is
DropletState startState = 6; // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
string qemuMachine = 7; // the qemu machine type to use "pc-q35-9.0"
int64 spicePort = 8; // preferred port to use for spice
string preferred_hypervisor = 9; // the hypervisor to prefer to run the droplet on
string force_hypervisor = 10; // use this hypervisor and this hypervisor only
string preferred_arch = 11; // the cpu arch to use "x86_64" (should really get this from the disk?)
repeated Network networks = 12; // really just mac addresses. should be unique across cluster
repeated Disk disks = 13; // disks to attach
string preferredHypervisor = 9; // the hypervisor to prefer to run the droplet on
string forceHypervisor = 10; // use this hypervisor and this hypervisor only
string preferredArch = 11; // the cpu arch to use "x86_64" (should really get this from the disk?)
repeated Network networks = 12; // really just mac addresses. should be unique across cluster
repeated Disk disks = 13; // disks to attach
string local_only = 14; // this is only defined locally on the hypervisor
string custom_xml = 15; // if needed,
Archive archive = 16; // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
string localOnly = 14; // this is only defined locally on the hypervisor
string customXml = 15; // if needed,
Archive archive = 16; // what the state of the droplet is SUPPOSED TO BE ('on' or 'off')
google.protobuf.Timestamp unstable = 39; // the last time we heard anything from this droplet
google.protobuf.Duration unstable_timeout = 40; // the last time we heard anything from this droplet
google.protobuf.Timestamp unstable = 39; // the last time we heard anything from this droplet
google.protobuf.Duration unstableTimeout = 40; // the last time we heard anything from this droplet
}
// volatile data. the current settings and values of things.
// These are passed around while the cluster to monitor and control the systems
// but they are not saved to the config file
message Current {
DropletState state = 1; // used to track the current state before taking any action
string hypervisor = 2; // the current hypervisor the droplet is running on
int64 start_attempts = 3; // how many times a start has been attempted
string full_xml = 4; // the full libvirt xml to import
google.protobuf.Timestamp last_poll = 5; // the last time we heard anything from this droplet
string image_url = 6; // url to the image
google.protobuf.Timestamp off_since = 7; // when the droplet was turned off
google.protobuf.Timestamp on_since = 8; // when the droplet was turned on
DropletState state = 1; // used to track the current state before taking any action
string hypervisor = 2; // the current hypervisor the droplet is running on
int64 startAttempts = 3; // how many times a start has been attempted
string fullXml = 4; // the full libvirt xml to import
google.protobuf.Timestamp lastPoll = 5; // the last time we heard anything from this droplet
string imageUrl = 6; // url to the image
google.protobuf.Timestamp offSince = 7; // when the droplet was turned off
google.protobuf.Timestamp onSince = 8; // when the droplet was turned on
}
message Archive {
DropletArchive reason = 1; // why the droplet was archived
google.protobuf.Timestamp when = 2; // when it was archived
DropletArchive reason = 1; // why the droplet was archived
google.protobuf.Timestamp when = 2; // when it was archived
}
// virtual machine state
enum DropletState {
ON = 0;
OFF = 1;
UNKNOWN = 2; // qemu says 'Shutdown'
PAUSED = 3;
CRASHED = 4;
INMIGRATE = 5;
ON = 0;
OFF = 1;
UNKNOWN = 2; // qemu says 'Shutdown'
PAUSED = 3;
CRASHED = 4;
INMIGRATE = 5;
}
enum DropletArchive {
DUP = 0;
USER = 1;
DUP = 0;
USER = 1;
}
message Network {
string mac = 1;
string name = 2;
string mac = 1;
string name = 2;
}
message Disk {
string filename = 1;
string filepath = 2;
int64 size = 3;
string qemu_arch = 4; // what arch. example: "x86_64" or "riscv64"
string filename = 1;
string filepath = 2;
int64 size = 3;
string qemuArch = 4; // what arch. example: "x86_64" or "riscv64"
}

View File

@ -1,14 +1,17 @@
syntax = "proto3";
package virtbuf;
package virtpb;
import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
import "google/protobuf/any.proto"; // Import the well-known type for Timestamp
import "droplet.proto";
message Events {
string uuid = 1; // I guess why not just have this on each file
string version = 2; // maybe can be used for protobuf schema change violations
int64 event_size = 3; // max events to store in a single
repeated Event events = 4; // all the events
// global settings for autogenpb `autogenpb:no-sort` `autogenpb:mutex`
message Events { // `autogenpb:marshal` `autogenpb:gui`
string uuid = 1; // `autogenpb:uuid:1e3a50c7-5916-4423-b33c-f0b977a7e446`
string version = 2; // `autogenpb:version:v0.0.1`
int64 eventSize = 3; // max events to store in a single
repeated Event events = 4; // all the events
}
// this information leans towards being human readable not programatic
@ -16,34 +19,42 @@ message Events {
// at least for now in the early days. but maybe forever.
// homelab clouds normally don't have many events.
// we are talking less than 1 a minute. even 1 an hour is often a lot
message Event {
int32 id = 1;
EventType etype = 2;
string droplet = 3; // name of the droplet
string droplet_uuid = 4; // uuid of the droplet
string hypervisor = 5; // name of the hypervisor
string hypervisor_uuid = 6; // uuid of the hypervisor
google.protobuf.Timestamp start = 7; // start time
google.protobuf.Timestamp end = 8; // end time
string field_name = 9; // the field name that changed
string orig_val = 10; // original value
string new_val = 11; // new value
google.protobuf.Any orig_any = 12; // anypb format. probably overkill
google.protobuf.Any new_any = 13; // anypb format
message Event { // `autogenpb:marshal`
enum status {
DONE = 0;
FAIL = 1;
RUNNING = 2;
}
int32 id = 1; // `autogenpb:unique` // should be unique across the cluster
EventType etype = 2;
string dropletName = 3; // name of the droplet
string dropletUuid = 4; // uuid of the droplet
string hypervisor = 5; // name of the hypervisor
string hypervisorUuid = 6; // uuid of the hypervisor
google.protobuf.Timestamp start = 7; // start time
google.protobuf.Timestamp end = 8; // end time
string fieldName = 9; // the field name that changed
string origVal = 10; // original value
string newVal = 11; // new value
google.protobuf.Any origAny = 12; // anypb format. probably overkill
google.protobuf.Any newAny = 13; // anypb format
string error = 14; // what went wrong
status state = 15; // state of the event
Droplet droplet = 16; // droplet
}
enum EventType {
ADD = 0;
DELETE = 1;
POWERON = 2;
POWEROFF = 3; // should indicate a "normal" shutdown
HIBERNATE = 4;
MIGRATE = 5;
DEMO = 6;
GET = 7; // request something
LOGIN = 8; // attempt to login
OK = 9; // everything is ok
FAIL = 10; // everything failed
CRASH = 11; // droplet hard crashed
CHANGE = 12; // droplet or hypervisor config change
ADD = 0;
DELETE = 1;
POWERON = 2;
POWEROFF = 3; // should indicate a "normal" shutdown
HIBERNATE = 4;
MIGRATE = 5;
DEMO = 6;
GET = 7; // request something
LOGIN = 8; // attempt to login
OK = 9; // everything is ok
FAIL = 10; // everything failed
CRASH = 11; // droplet hard crashed
CHANGE = 12; // droplet or hypervisor config change
EDIT = 13; // edit droplet settings
}

View File

@ -1,15 +0,0 @@
build:
GO111MODULE=off go build
./example
goimports:
goimports -w *.go
prep:
go get -v -t -u
run:
go run *.go
clean:
-rm -f example

View File

@ -1,91 +0,0 @@
package main
import (
"fmt"
"os"
pb "go.wit.com/lib/protobuf/virtbuf"
)
//
// saves entries in a config file
//
func main() {
TestWriteCluster()
var c *pb.NewCluster
c = pb.InitCluster()
// log.Println(aCluster.String())
// show the droplets to STDOUT
loop := c.DropletsAll() // get the list of droplets
for loop.Scan() {
d := loop.Droplet()
fmt.Println("\tdroplet =", d.Hostname, "preffered host:", d.PreferredHypervisor)
}
/*
// show the hypervisors to STDOUT
for _, h := range aCluster.Hypervisors {
fmt.Println("\thypervisor =", h.Hostname, h.GetMemoryPrintable())
}
*/
/*
json := aCluster.FormatJSON()
fmt.Println(json)
data, _ := aCluster.MarshalJSON()
fmt.Println(string(data))
text := aCluster.FormatTEXT()
fmt.Println(text)
*/
}
/*
func marshalWriteToFile(myWriter *bufio.Writer, c *pb.NewCluster) {
buf, err := proto.Marshal(c)
if err != nil {
log.Fatal("marshaling error: ", err)
}
tmp, err := myWriter.Write(buf)
myWriter.Flush()
log.Println("bufio.Write() tmp, err = ", tmp, err)
buf, err = proto.Marshal(c)
tmp2, err := myWriter.Write(buf)
myWriter.Flush()
log.Println("bufio.Write() tmp2, err = ", tmp2, err)
}
*/
func TestWriteCluster() {
c := pb.CreateSampleCluster(7)
os.Setenv("VIRTIGO_HOME", "/tmp/virtigo/")
if err := c.ConfigSave(); err != nil {
fmt.Println("configsave error", err)
os.Exit(-1)
}
// marshalUnmarshal()
}
/*
func marshalUnmarshal() {
test := pb.CreateSampleCluster(7)
data, err := proto.Marshal(test)
if err != nil {
log.Fatal("marshaling error: ", err)
}
newTest := &pb.Cluster{}
err = proto.Unmarshal(data, newTest)
if err != nil {
log.Fatal("unmarshaling error: ", err)
} else {
log.Println("proto.Marshal() and proto.Unmarshal() worked")
}
}
*/

View File

@ -1,19 +0,0 @@
syntax = "proto3";
package virtbuf;
import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
import "google/protobuf/any.proto"; // Import the well-known type for Timestamp
message WhatsThis {
// is it possible to have custom formatting in JSON and TEXT marshal/unmarshal ?
WhatInfo humantest = 1;
google.protobuf.Timestamp end = 2; // end time
google.protobuf.Any orig_val = 3; // original value
google.protobuf.Any new_val = 4; // new value
}
// this is for exerimenting
message WhatInfo {
int64 capacity = 1; // Stores the storage capacity in bytes.
}

View File

@ -1,81 +1,17 @@
package virtbuf
package virtpb
// functions to import and export the protobuf
// data to and from config files
import (
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/encoding/prototext"
"google.golang.org/protobuf/proto"
)
func InitCluster() *NewCluster {
var c *NewCluster
c = new(NewCluster)
func InitCluster() *OldCluster {
var c *OldCluster
c = new(OldCluster)
c.d = new(Droplets)
c.H = new(Hypervisors)
c.e = new(Events)
return c
}
// human readable JSON
func (d *Droplets) FormatJSON() string {
return protojson.Format(d)
}
func (d *Droplet) FormatJSON() string {
return protojson.Format(d)
}
func (e *Events) FormatJSON() string {
return protojson.Format(e)
}
func (h *Hypervisors) FormatJSON() string {
return protojson.Format(h)
}
// apparently this isn't supposed to be used?
// https://protobuf.dev/reference/go/faq/#unstable-text
// this is a shame because this is much nicer output than JSON Format()
func (d *Droplets) FormatTEXT() string {
return prototext.Format(d)
}
func (d *Droplet) FormatTEXT() string {
return prototext.Format(d)
}
func (e *Events) FormatTEXT() string {
return prototext.Format(e)
}
// marshal
func (d *Droplets) MarshalJSON() ([]byte, error) {
return protojson.Marshal(d)
}
func (d *Droplet) MarshalJSON() ([]byte, error) {
return protojson.Marshal(d)
}
func (e *Events) MarshalJSON() ([]byte, error) {
return protojson.Marshal(e)
}
// unmarshal
func (d *Droplets) UnmarshalJSON(data []byte) error {
return protojson.Unmarshal(data, d)
}
func (d *Droplet) UnmarshalJSON(data []byte) error {
return protojson.Unmarshal(data, d)
}
func (e *Events) UnmarshalJSON(data []byte) error {
return protojson.Unmarshal(data, e)
}
func (d *Droplet) Unmarshal(data []byte) error {
return proto.Unmarshal(data, d)
func (c *OldCluster) DropletsAll() *DropletScanner {
return c.d.All()
}

View File

@ -1,9 +1,9 @@
package virtbuf
package virtpb
// mostly just functions related to making STDOUT
// more readable by us humans
// also function shortcuts the do limited formatting (haha, who remembers COBOL?)
// also function shortcuts the do fixed limited formatting (it's like COBOL)
// so reporting tables of the status of what droplets and hypervisors
// are in text columns and rows that can be easily read in a terminal
@ -14,6 +14,8 @@ import (
"strings"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
"go.wit.com/log"
)
@ -95,7 +97,7 @@ func FormatDuration(d time.Duration) string {
ms := int(d.Milliseconds())
if ms > 100 {
// todo: print .3s, etc ?
return fmt.Sprintf("%1.2fs", seconds/1000)
return fmt.Sprintf("%1.2fs", float64(seconds)/1000)
}
if ms > 0 {
result += fmt.Sprintf("%dms", ms)
@ -146,34 +148,26 @@ func (d *Droplet) SprintDumpHeader() string {
macs = append(macs, n.Mac)
}
// this line in golang could replace 80 lines of COBOL
header := fmt.Sprintf("%-4.4s%20s %-8s", d.Current.State, strings.Join(macs, " "), d.Current.Hypervisor)
if d.Current == nil {
d.Current = new(Current)
return header
}
if d.Current.OnSince == nil {
d.Current.OnSince = timestamppb.New(time.Now())
}
t := time.Since(d.Current.OnSince.AsTime()) // time since 'ON'
dur := FormatDuration(t)
switch d.Current.State {
case DropletState_ON:
var dur string
if d.Current.OnSince != nil {
dur = ""
} else {
t := time.Since(d.Current.OnSince.AsTime()) // time since 'ON'
dur = FormatDuration(t)
}
header += fmt.Sprintf(" (on :%3s)", dur)
case DropletState_OFF:
var dur string
if d.Current.OffSince != nil {
dur = ""
} else {
t := time.Since(d.Current.OffSince.AsTime()) // time since 'OFF'
dur = FormatDuration(t)
}
header += fmt.Sprintf(" (off:%3s)", dur)
default:
header += fmt.Sprintf(" (?? :%3s)", "")
header += fmt.Sprintf(" (?? :%3s)", dur)
}
return header
}
@ -191,7 +185,7 @@ func (d *Droplet) DumpDroplet(w http.ResponseWriter, r *http.Request) (string, e
return t, nil
}
func (c *NewCluster) DumpDroplet(w http.ResponseWriter, r *http.Request) (string, error) {
func (c *OldCluster) DumpDroplet(w http.ResponseWriter, r *http.Request) (string, error) {
hostname := r.URL.Query().Get("hostname")
d := c.FindDropletByName(hostname)
if d == nil {

View File

@ -1,26 +1,29 @@
syntax = "proto3";
package virtbuf;
package virtpb;
message Hypervisors {
string uuid = 1; // I guess why not just have this on each file
string version = 2; // maybe can be used for protobuf schema change violations
repeated Hypervisor hypervisors = 3;
import "google/protobuf/timestamp.proto";
message Hypervisors { // `autogenpb:marshal` `autogenpb:gui`
string uuid = 1; // `autogenpb:uuid:6e3aa8b9-cf98-40f6-af58-3c6ad1edf4d4`
string version = 2; // `autogenpb:version:v0.0.1`
repeated Hypervisor hypervisors = 3;
}
message Hypervisor {
string uuid = 1;
string hostname = 2;
bool active = 3; // is allowed to start new droplets
int64 cpus = 4;
int64 memory = 5; // in bytes
string comment = 6;
bool autoscan = 7; // to scan or not to scan by virtigo
HypervisorArch arch = 8;
string uuid = 1; // `autogenpb:unique`
string hostname = 2; // `autogenpb:unique`
bool active = 3; // is allowed to start new droplets
int64 cpus = 4;
int64 memory = 5; // in bytes
string comment = 6;
bool autoscan = 7; // to scan or not to scan by virtigo
HypervisorArch arch = 8;
int64 killcount = 9; // in bytes
google.protobuf.Timestamp lastPoll = 10; // the last time we heard anything
}
// think about this more
enum HypervisorArch {
RISCV64 = 0;
X86_64 = 1;
ARM64 = 2;
RISCV64 = 0;
X86_64 = 1;
ARM64 = 2;
}

View File

@ -1,19 +0,0 @@
package virtbuf
import (
sync "sync"
durationpb "google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
)
type NewCluster struct {
sync.RWMutex
Dirs []string
d *Droplets
H *Hypervisors
e *Events
Unstable *timestamppb.Timestamp
UnstableTimeout *durationpb.Duration
}

48
oldCluster.go Normal file
View File

@ -0,0 +1,48 @@
package virtpb
import (
sync "sync"
durationpb "google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
)
type OldCluster struct {
sync.RWMutex
Dirs []string
d *Droplets
H *Hypervisors
e *Events
Unstable *timestamppb.Timestamp
UnstableTimeout *durationpb.Duration
}
func (c *OldCluster) GetDropletsPB() *Droplets {
return c.d
}
func (c *OldCluster) GetHypervisorsPB() *Hypervisors {
return c.H
}
func (c *OldCluster) GetEventsPB() *Events {
return c.e
}
// adds a new droplet. enforce unique hostnames
func (c *OldCluster) AddDroplet(newd *Droplet) bool {
c.Lock()
defer c.Unlock()
for _, d := range c.d.Droplets {
if newd.Hostname == d.Hostname {
// boo. that one is already here
return false
}
}
// everything is ok, this hostname is new
c.d.Droplets = append(c.d.Droplets, newd)
return true
}

View File

@ -1,4 +1,4 @@
package virtbuf
package virtpb
import (
"fmt"
@ -39,22 +39,6 @@ func CreateSampleHypervisor(hostname string, mem int) *Hypervisor {
return h
}
func CreateExperiment(total int) *WhatsThis {
var e *WhatsThis
e = new(WhatsThis)
// info := StorageInfo{Capacity: 64}
// e.Humantest = &info
if e.Humantest == nil {
var newInfo WhatInfo
newInfo = WhatInfo{Capacity: 64}
e.Humantest = &newInfo
} else {
e.Humantest.Capacity = SetGB(total * 32)
}
return e
}
func CreateSampleEvents(total int) *Events {
var e *Events
e = new(Events)
@ -67,7 +51,7 @@ func CreateSampleEvents(total int) *Events {
return e
}
func CreateSampleCluster(total int) *NewCluster {
func CreateSampleCluster(total int) *OldCluster {
c := InitCluster()
for i := 0; i < total; i++ {

View File

@ -1,106 +0,0 @@
package virtbuf
import (
"fmt"
"os"
"go.wit.com/log"
)
type DropletIterator struct {
droplets []*Droplet
index int
}
// NewDropletIterator initializes a new iterator.
func NewDropletIterator(droplets []*Droplet) *DropletIterator {
return &DropletIterator{droplets: droplets}
}
// Scan moves to the next element and returns false if there are no more droplets.
func (it *DropletIterator) Scan() bool {
if it.index >= len(it.droplets) {
return false
}
it.index++
return true
}
// Droplet returns the current droplet.
func (it *DropletIterator) Droplet() *Droplet {
if it.droplets[it.index-1] == nil {
for i, d := range it.droplets {
fmt.Println("i =", i, d)
}
fmt.Println("len =", len(it.droplets))
fmt.Println("droplet == nil", it.index, it.index-1)
os.Exit(-1)
}
return it.droplets[it.index-1]
}
// Use Scan() in a loop, similar to a while loop
//
// for iterator.Scan() {
// d := iterator.Droplet()
// fmt.Println("Droplet UUID:", d.Uuid)
// }
func (c *NewCluster) GetDropletIterator() *DropletIterator {
dropletPointers := c.SelectDropletPointers()
iterator := NewDropletIterator(dropletPointers)
return iterator
}
func (c *NewCluster) DropletsAll() *DropletIterator {
dropletPointers := c.SelectDropletAll()
iterator := NewDropletIterator(dropletPointers)
return iterator
}
// SelectDropletPointers safely returns a slice of pointers to Droplet records.
func (c *NewCluster) SelectDropletAll() []*Droplet {
c.RLock()
defer c.RUnlock()
// Create a new slice to hold pointers to each Droplet
// dropletPointers := make([]*Droplet, len(c.E.Droplets))
var dropletPointers []*Droplet
if c.d == nil {
log.Info("SelectDropletsAll() c.d == nil")
// os.Exit(-1)
}
for _, d := range c.d.Droplets {
if d == nil {
continue
}
if d.Archive != nil {
continue
}
dropletPointers = append(dropletPointers, d) // Copy pointers for safe iteration
}
return dropletPointers
}
// SelectDropletPointers safely returns a slice of pointers to Droplet records.
func (c *NewCluster) SelectDropletPointers() []*Droplet {
c.RLock()
defer c.RUnlock()
// Create a new slice to hold pointers to each Droplet
// dropletPointers := make([]*Droplet, len(c.E.Droplets))
dropletPointers := make([]*Droplet, 1)
if c.d == nil {
log.Info("c.d == nil")
os.Exit(-1)
}
for _, d := range c.d.Droplets {
dropletPointers = append(dropletPointers, d) // Copy pointers for safe iteration
}
return dropletPointers
}

View File

@ -1,11 +1,6 @@
package virtbuf
import (
"encoding/json"
"fmt"
"strconv"
)
package virtpb
/*
// MarshalJSON custom marshals the WhatInfo struct to JSON
func (s WhatInfo) MarshalJSON() ([]byte, error) {
capacityStr := fmt.Sprintf("%d GB", s.Capacity)
@ -37,7 +32,6 @@ func (s *WhatInfo) UnmarshalJSON(data []byte) error {
return nil
}
/*
func main() {
info := WhatInfo{Capacity: 64}