make something to save the books.pb

Makefile

@@ -1,4 +1,4 @@
-all: clean chat.pb.go goimports vet
+all: clean chat.pb.go book.pb.go goimports vet
 
 goimports:
 	goimports -w *.go
@@ -6,6 +6,9 @@ goimports:
 chat.pb.go: chat.proto
 	autogenpb --proto chat.proto
 
+book.pb.go: book.proto
+	autogenpb --proto book.proto
+
 clean:
 	rm -f *.pb.go *.patch
 	-rm -f go.*

addChat.go

@@ -0,0 +1,35 @@
+// Copyright 2025 WIT.COM Inc Licensed GPL 3.0
+
+package chatpb
+
+import (
+	"time"
+
+	"go.wit.com/log"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+)
+
+// returns true if the pb was added
+// to indicate that ConfigSave() should be run to write it out to disk
+func (c *Chat) AddGeminiRequest(fname string, age time.Time, pb *GeminiRequest) bool {
+	for _, e := range c.GetEntries() {
+		if e.GetContentFile() == fname {
+			log.Info("fname already here", fname)
+			if iContent, iParts, ok := e.VerifyGeminiRequest(pb); ok {
+				log.Info("pb is already here with same size", iContent, iParts)
+				return false
+			} else {
+				log.Info("pb is already here but things don't match", iContent, iParts)
+			}
+			return false
+		}
+	}
+	log.Info("Adding new ChatEntry for", "/tmp/"+fname)
+	e := new(ChatEntry)
+	e.Ctime = timestamppb.New(age)
+	e.From = Who_USER
+	e.ContentFile = fname
+	e.GeminiRequest = pb
+	c.AppendEntry(e)
+	return true
+}

book.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package chatpb;
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/struct.proto";
+import "chat.proto";
+
+message Book {
+        string                      uuid              = 1;  // `autogenpb:unique` `autogenpb:sort`
+        google.protobuf.Timestamp   ctime             = 2;
+        string                      Title             = 3;
+        int32                       version           = 4;
+        Who                         from              = 5;
+        string                      content           = 6;
+        Table                       table             = 7;
+        GeminiRequest               GeminiRequest     = 8;
+}
+
+message Books {                                             // `autogenpb:marshal` `autogenpb:mutex` `autogenpb:gui`
+        string                      uuid              = 1;  // `autogenpb:uuid:8b6409ad-4498-43a6-b09a-7835c00dcb9a`
+        string                      version           = 2;  // `autogenpb:version:v0.0.1`
+        repeated Book               Books             = 3;  // THIS MUST BE  Chat  and then  Chats
+        google.protobuf.Timestamp   ctime             = 4;
+        string                      Title             = 5;
+        string                      TitleUuid         = 6;
+}

chat.proto

@@ -36,14 +36,19 @@ message FunctionDeclaration {
         Schema                                     parameters_json_schema            = 3;
 }
 
+message GoogleSearch {
+}
+
 message Tool {
         repeated FunctionDeclaration               functionDeclarations              = 1;
+        GoogleSearch                               googleSearch                      = 2;
 }
 
 // Configuration for the request
 message Config {
         message ThinkingConfig {
                 bool    includeThoughts     = 1;
+                int32   thinkingBudget      = 2;
         }
         double                                     temperature                       = 2;
         double                                     topP                              = 3;
@@ -65,9 +70,27 @@ message ResponseJsonSchema {
 message Properties {
         Reasoning                                  reasoning                         = 1;
         NextSpeaker                                next_speaker                      = 2;
+        CorrectedNewStringEscaping                 corrected_new_string_escaping     = 3;
+        CorrectedTargetSnippet                     corrected_target_snippet          = 4;
+        Confidence                                 confidence                        = 5;
+}
+
+message Confidence {
+        string                                     type                              = 1;
+        string                                     description                       = 2;
+}
+
+message CorrectedTargetSnippet {
+        string                                     type                              = 1;
+        string                                     description                       = 2;
 }
 
 // Reasoning property
+message CorrectedNewStringEscaping {
+        string                                     type                              = 1;
+        string                                     description                       = 2;
+}
+
 message Reasoning {
         string                                     type                              = 1;
         string                                     description                       = 2;
@@ -115,6 +138,8 @@ message argsInfo {
         string                                     pattern                           = 11;
         string                                     content                           = 12;
         string                                     fact                              = 13;
+        repeated string                            paths                             = 14;
+        string                                     query                             = 15;
 }
 
 // Function response
@@ -159,7 +184,7 @@ message CodeSnippet {
         string                                     content                           = 2;
 }
 
-message ChatEntry {
+message ChatEntry {                                                                        // `autogenpb:marshal`
         Who                                        from                              = 1;
         google.protobuf.Timestamp                  ctime                             = 2;
         string                                     content                           = 3;
@@ -171,6 +196,7 @@ message ChatEntry {
         repeated Part                              parts                             = 9;
         GeminiRequest                              GeminiRequest                     = 10;
         int32                                      RequestCounter                    = 11;
+        repeated string                            paths                             = 12;
 }
 
 message SessionStats {
@@ -185,7 +211,7 @@ message Chat {
         repeated SessionStats                      Session                           = 5;
 }
 
-message Chats {                                                                     // `autogenpb:marshal` `autogenpb:mutex`
+message Chats {                                                                            // `autogenpb:marshal` `autogenpb:mutex` `autogenpb:gui`
         string                                     uuid                              = 1;  // `autogenpb:uuid:9fd31f10-c25d-4d66-bc8d-5f6eb7c79057` `autogenpb:primary`
         string                                     version                           = 2;  // `autogenpb:version:v0.0.1`
         repeated Chat                              Chats                             = 3;  // THIS MUST BE  Chat  and then  Chats

config.books.go

@@ -0,0 +1,78 @@
+package chatpb
+
+// functions to import and export the protobuf
+// data to and from config files
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"go.wit.com/lib/protobuf/bugpb"
+	"go.wit.com/log"
+)
+
+// write to ~/.config/regex/ unless ENV{REGEX_HOME} is set
+func (all *Books) ConfigSave() error {
+	if os.Getenv("REGEX_HOME") == "" {
+		homeDir, _ := os.UserHomeDir()
+		fullpath := filepath.Join(homeDir, ".config/regex")
+		os.Setenv("REGEX_HOME", fullpath)
+	}
+	if all == nil {
+		log.Warn("chatpb all == nil")
+		return errors.New("chatpb.Books.ConfigSave() all == nil")
+	}
+	log.Info("Books.ConfigSave()")
+
+	// --- Start of Fix ---
+	// Create a new, clean Books object to avoid marshaling a slice with nil entries.
+	cleanBooks := NewBooks() // Assuming NewBooks() initializes the struct correctly.
+
+	// Loop through the original books and append only the non-nil ones.
+	for _, book := range all.GetBooks() {
+		if book != nil {
+			cleanBooks.Books = append(cleanBooks.Books, book)
+		} else {
+			log.Warn("Found and skipped a nil book entry during Books.ConfigSave")
+		}
+	}
+	// --- End of Fix ---
+
+	data, err := cleanBooks.Marshal() // Marshal the clean object, not 'all'
+	if err != nil {
+		log.Info("chatpb proto.Marshal() failed len", len(data), err)
+		// The tryValidate logic might be less necessary now but kept for safety.
+		if err := cleanBooks.tryValidate(); err != nil {
+			return err
+		} else {
+			data, err = cleanBooks.Marshal() // Retry with the clean object
+			if err == nil {
+				log.Info("chatpb.Books.ConfigSave() pb.Marshal() worked after validation len", len(cleanBooks.Books), "books")
+				configWrite("regex.pb", data)
+				return nil
+			}
+		}
+		return err
+	}
+
+	filename := "book." + all.GetTitleUuid() + ".pb"
+	if err := configWrite(filename, data); err != nil {
+		log.Infof("chatpb.Books.ConfigSave() failed len(Books)=%d bytes=%d", len(cleanBooks.Books), len(data))
+		return err
+	}
+	return nil
+}
+
+func (all *Books) tryValidate() error {
+	err := bugpb.ValidateProtoUTF8(all)
+	if err != nil {
+		log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
+	}
+	if err := bugpb.SanitizeProtoUTF8(all); err != nil {
+		log.Warn("Sanitation failed:", err)
+		// log.Fatalf("Sanitization failed: %v", err)
+		return err
+	}
+	return nil
+}

config.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"go.wit.com/lib/gui/shell"
 	"go.wit.com/lib/protobuf/bugpb"
 	"go.wit.com/log"
 )
@@ -25,6 +26,7 @@ func (all *Chats) ConfigSave() error {
 		log.Warn("chatpb all == nil")
 		return errors.New("chatpb.ConfigSave() all == nil")
 	}
+	log.Info("DOING ConfigSave()")
 
 	// --- Start of Fix ---
 	// Create a new, clean Chats object to avoid marshaling a slice with nil entries.
@@ -46,7 +48,7 @@ func (all *Chats) ConfigSave() error {
 	if err != nil {
 		log.Info("chatpb proto.Marshal() failed len", len(data), err)
 		// The tryValidate logic might be less necessary now but kept for safety.
-		if err := all.tryValidate(); err != nil {
+		if err := cleanChats.tryValidate(); err != nil {
 			return err
 		} else {
 			data, err = cleanChats.Marshal() // Retry with the clean object
@@ -61,6 +63,19 @@ func (all *Chats) ConfigSave() error {
 
 	// --- Backup Logic ---
 	filename := filepath.Join(os.Getenv("REGEX_HOME"), "regex.pb")
+	filebackup := filepath.Join(os.Getenv("REGEX_HOME"), "regex.backup.pb")
+	if s, err := os.Stat(filebackup); err == nil {
+		log.Info("STAT OF CONFIG BACKUP WORKED", filebackup)
+		age := time.Since(s.ModTime())
+		if age > 10*time.Second {
+			log.Info(filebackup, "is greater than 10 minutes")
+			shell.RunVerbose([]string{"cp", filename, filebackup})
+		} else {
+			log.Info(filebackup, "is less than 10 minutes")
+		}
+	} else {
+		log.Info("STAT OF CONFIG BACKUP FAILED", filebackup)
+	}
 	if _, err := os.Stat(filename); err == nil {
 		// File exists, so back it up.
 		dir := filepath.Dir(filename)
@@ -77,13 +92,12 @@ func (all *Chats) ConfigSave() error {
 		log.Infof("chatpb.ConfigSave() failed len(Chats)=%d bytes=%d", len(cleanChats.Chats), len(data))
 		return err
 	}
-	configWrite("regex.text", []byte(cleanChats.FormatTEXT()))
+	// configWrite("regex.text", []byte(cleanChats.FormatTEXT()))
 	// log.Infof("chatpb.ConfigSave() worked len(Chats)=%d bytes=%d", len(cleanChats.Chats), len(data))
 	return nil
 }
 
 func (all *Chats) tryValidate() error {
-
 	err := bugpb.ValidateProtoUTF8(all)
 	if err != nil {
 		log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
@@ -169,15 +183,6 @@ func configWrite(filename string, data []byte) error {
 		log.Warn("open config file :", err)
 		return err
 	}
-	if filename == "regex.text" {
-		// add header
-		cfgfile.Write([]byte("# this file is automatically re-generated from regex.pb, however,\n"))
-		cfgfile.Write([]byte("# if you want to edit it by hand, you can:\n"))
-		cfgfile.Write([]byte("# stop regex; remove regex.pb; edit regex.text; start regex\n"))
-		cfgfile.Write([]byte("# this will cause the default behavior to fallback to parsing this file for the config\n"))
-		cfgfile.Write([]byte("\n"))
-		cfgfile.Write([]byte("# this file is intended to be used to customize settings on what\n"))
-	}
 	cfgfile.Write(data)
 	return nil
 }

find.go

@@ -0,0 +1,24 @@
+// Copyright 2025 WIT.COM Inc Licensed GPL 3.0
+
+package chatpb
+
+func (all *Chats) FindUuid(id string) *Chat {
+	for chat := range all.IterAll() {
+		if chat.Uuid == id {
+			return chat
+		}
+
+		for _, e := range chat.GetSession() {
+			if id == e.Uuid {
+				return chat
+			}
+		}
+
+		for _, e := range chat.GetEntries() {
+			if id == e.Uuid {
+				return chat
+			}
+		}
+	}
+	return nil
+}

helpers.go

@@ -101,3 +101,35 @@ func (x *Chat) AppendEntry(y *ChatEntry) {
 
 	x.Entries = append(x.Entries, proto.Clone(y).(*ChatEntry))
 }
+
+// returns true if the pb is probably the same
+func (c *ChatEntry) VerifyGeminiRequest(pb *GeminiRequest) (int, int, bool) {
+	if c.GeminiRequest == nil {
+		log.Warn("There is no GeminiRequest in the chat protobuf")
+		return -1, -1, false
+	}
+	if pb == nil {
+		log.Warn("There is no GeminiRequest in the passed in protobuf")
+		return -1, -1, false
+	}
+	if len(c.GeminiRequest.GetContents()) != len(pb.GetContents()) {
+		log.Warn("c != pb", len(c.GeminiRequest.GetContents()), len(pb.GetContents()))
+		return -1, -1, false
+	}
+	var cCount int  // # of Parts in all the Contents
+	var pbCount int // # of Parts in all the Contents
+	for _, grc := range c.GeminiRequest.GetContents() {
+		cCount += len(grc.GetParts())
+
+	}
+	for _, grc := range pb.GetContents() {
+		pbCount += len(grc.GetParts())
+
+	}
+	if cCount != pbCount {
+		log.Warn("c != pb", cCount, pbCount)
+		return len(c.GeminiRequest.GetContents()), cCount, false
+	}
+	log.Info("EVERYTHING MATCHED", cCount, pbCount, len(c.GeminiRequest.GetContents()), len(pb.GetContents()))
+	return len(c.GeminiRequest.GetContents()), cCount, true
+}

humanTable.go

@@ -0,0 +1,157 @@
+// Copyright 2025 WIT.COM Inc Licensed GPL 3.0
+
+package chatpb
+
+import (
+	"strings"
+	"time"
+
+	"go.wit.com/lib/cobol"
+	"go.wit.com/lib/gui/shell"
+	"go.wit.com/log"
+)
+
+func (all *Chats) PrintHumanTable() {
+	log.DaemonMode(true)
+
+	// print the header
+	args := []string{"uuid", "name", "age", "master", "devel", "user", "curver", "lasttag", "next", "repo type"}
+	sizes := []int{40, 40, 6, 4, 4, 4, 4, 4, 4, 4}
+	log.Info(cobol.StandardTableSize10(sizes, args))
+
+	for chat := range all.IterAll() {
+		chat.printChatToTable(sizes)
+	}
+	log.Infof("Total Chats: %d\n", all.Len())
+}
+
+func (c *Chat) printChatToTable(sizes []int) {
+	var args []string
+	age := c.Ctime.AsTime().String()
+	args = []string{c.Uuid, age, c.GetChatName(), "", "", "", "", "", "", ""}
+
+	start := cobol.StandardTableSize10(sizes, args)
+
+	log.Info(start)
+}
+
+func (c *Chat) PrintChatStatsTable() {
+	log.DaemonMode(true)
+
+	// print the header
+	args := []string{"uuid", "name", "age", "master", "devel", "user", "curver", "lasttag", "next", "repo type"}
+	sizes := []int{40, 40, 6, 4, 4, 4, 4, 4, 4, 4}
+	log.Info(cobol.StandardTableSize10(sizes, args))
+
+	for _, e := range c.GetSession() {
+		var args []string
+		args = []string{e.Uuid, "", "", "", "", "", "", "", "", ""}
+
+		start := cobol.StandardTableSize10(sizes, args)
+
+		log.Info(start)
+	}
+	log.Infof("Total Chats: %d\n", len(c.GetEntries()))
+}
+
+func (c *Chat) PrintChatEntriesTable() {
+	log.DaemonMode(true)
+
+	// print the header
+	args := []string{"uuid", "age", "con file", "Who", "model", "", "", "", "", ""}
+	sizes := []int{40, 16, 8, 4, 8, 2, 2, 2, 2, 2}
+	log.Info(cobol.StandardTableSize10(sizes, args))
+
+	for _, e := range c.GetEntries() {
+		var args []string
+		age := e.Ctime.AsTime().String()
+		args = []string{e.Uuid, age, e.GetContentFile(), e.From.String(), "", "", "", "", "", ""}
+
+		start := cobol.StandardTableSize10(sizes, args)
+		log.Info(start)
+	}
+	log.Infof("Total Chats: %d\n", len(c.GetEntries()))
+}
+
+func (c *Chat) PrintChatGeminiTable() {
+	log.DaemonMode(true)
+
+	// print the header
+	args := []string{"uuid", "age", "ID", "Who", "model", "", "", "", "", ""}
+	sizes := []int{40, 5, 5, 8, 16, 2, 2, 2, 2, 2}
+	log.Info(cobol.StandardTableSize10(sizes, args))
+
+	for _, e := range c.GetEntries() {
+		var args []string
+		dur := time.Since(e.Ctime.AsTime())
+		age := shell.FormatDuration(dur)
+		var model string
+		var id string
+		if e.GeminiRequest == nil {
+			model = "nil"
+		} else {
+			model = e.GeminiRequest.Model
+		}
+		if e.GetContentFile() != "" {
+			parts := strings.Split(e.GetContentFile(), ".")
+			if len(parts) < 4 {
+				id = "??"
+			} else {
+				id = parts[3]
+			}
+		}
+		args = []string{e.Uuid, age, id, e.From.String(), model, "", "", "", "", ""}
+
+		start := cobol.StandardTableSize10(sizes, args)
+		log.Info(start, e.GetContentFile())
+	}
+	log.Infof("Total Chats: %d\n", len(c.GetEntries()))
+}
+
+func (gr *GeminiRequest) PrintGeminiTable() {
+	if gr == nil {
+		return
+	}
+	log.DaemonMode(true)
+
+	// print the header
+	args := []string{"model", "what", "age", "cId", "partId", "", "", "", "", ""}
+	sizes := []int{16, 8, 4, 4, 4, 16, 120, 2, 2, 2}
+	log.Info(cobol.StandardTableSize10(sizes, args))
+
+	var countCONTENTS int
+	var countPARTS int
+	for x, c := range gr.GetContents() {
+		model := gr.Model
+		cId := log.Sprintf("%d", x)
+		countCONTENTS += 1
+		for i, p := range c.GetParts() {
+			var args []string
+			partId := log.Sprintf("%d", i)
+
+			// removes newline chars from the text
+			parts := strings.Split(p.GetText(), "\n")
+			txt := strings.Join(parts, "__")
+
+			what := "TXT"
+
+			if fc := p.GetFunctionCall(); fc != nil {
+				what = "FuncCall"
+				txt = log.Sprintf("%s, %v", fc.Name, fc.Args)
+
+			}
+			if fr := p.GetFunctionResponse(); fr != nil {
+				what = "FuncResp"
+				txt = fr.Name
+				txt = log.Sprintf("%s %s, %v", fr.Id, fr.Name, fr.Response)
+			}
+
+			args = []string{model, what, "", cId, partId, p.ThoughtSignature, txt, "", "", ""}
+
+			start := cobol.StandardTableSize10(sizes, args)
+			log.Info(start)
+			countPARTS += 1
+		}
+	}
+	log.Infof("Total Contents (%d) Parts (%d)\n", countCONTENTS, countPARTS)
+}