notes for the future

.gitignore

@@ -1,5 +1,7 @@
-go.*
+*.swp
+*.patch
 
+go.*
 *.pb.go
 
 scanGoSrc/scanGoSrc

Makefile

@@ -19,10 +19,10 @@ vet:
 goimports:
 	goimports -w *.go
 
+# dump autogenerated files and potential patches
 clean:
-	rm -f *.pb.go
-	-rm -f go.*
-	go-mod-clean --purge
+	rm -f *.pb.go go.* *.patch
+	go-mod-clean purge
 
 #refs.pb.go: refs.proto
 #	cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \
@@ -37,3 +37,32 @@ goDep.pb.go: goDep.proto
 
 repo.pb.go: repo.proto
 	autogenpb --proto repo.proto
+
+protoc-bad:
+	cd ~/go/src && protoc \
+		--proto_path=. \
+		--proto_path=go.wit.com/lib/protobuf/gitpb \
+		--go_out=. \
+		--go_opt=Mgo.wit.com/lib/protobuf/gitpb/repo.proto=go.wit.com/lib/protobuf/gitpb \
+		--go_opt=MgitTag.proto=go.wit.com/lib/protobuf/gitpb \
+		--go_opt=MgoDep.proto=go.wit.com/lib/protobuf/gitpb \
+		go.wit.com/lib/protobuf/gitpb/repo.proto
+
+protoc-good:
+	cd ~/go/src && protoc \
+		--proto_path=. \
+		--go_out=go.wit.com/lib/protobuf/gitpb \
+		--go_opt=Mrepo.proto=go.wit.com/lib/protobuf/gitpb \
+		--go_opt=MgitTag.proto=go.wit.com/lib/protobuf/gitpb \
+		--go_opt=MgoDep.proto=go.wit.com/lib/protobuf/gitpb \
+		go.wit.com/lib/protobuf/gitpb/repo.proto
+
+protoc-todo-move-to-this:
+	# I think I should seperate these dirs. ONLY ONE .proto FILE PER DIRECTORY
+	# - httppb.HttpRequest     httpRequest     = 4;  // correct syntax
+	protoc \
+	    --proto_path=. \
+	    --go_out=. \
+	    --go_opt=Mgo.wit.com/lib/protobuf/forgepb/patchset.proto=go.wit.com/lib/protobuf/forgepb \
+	    --go_opt=Mgo.wit.com/lib/protobuf/httppb/httpRequest.proto=go.wit.com/lib/protobuf/httppb \
+	    go.wit.com/lib/protobuf/forgepb/patchset.proto

autogen.go

@@ -19,11 +19,11 @@ import (
 func (repo *Repo) AutogenSave(files []string, refname string, del bool) error {
 	if del {
 		cmd := []string{"git", "notes", "show", refname}
-		if err := repo.StrictRun(cmd); err != nil {
+		if _, err := repo.RunQuiet(cmd); err != nil {
 			// if there are not any notes, no need to remove them
 		} else {
 			cmd := []string{"git", "notes", "remove", refname}
-			if err := repo.StrictRun(cmd); err != nil {
+			if _, err := repo.RunQuiet(cmd); err != nil {
 				return err
 			}
 		}
@@ -31,18 +31,18 @@ func (repo *Repo) AutogenSave(files []string, refname string, del bool) error {
 	for _, fname := range files {
 		autotag := "// `autogen:" + fname + "`"
 		cmd := []string{"git", "notes", "append", "-m", autotag, refname}
-		if err := repo.StrictRun(cmd); err != nil {
+		if _, err := repo.RunQuiet(cmd); err != nil {
 			return err
 		}
 		cmd = []string{"git", "notes", "append", "-F", fname, refname}
-		if err := repo.StrictRun(cmd); err != nil {
+		if _, err := repo.RunQuiet(cmd); err != nil {
 			return err
 		}
 	}
 	// a tag with a blank name indicates the end of the autogen file or files
 	autotag := "// `autogen:`"
 	cmd := []string{"git", "notes", "append", "-m", autotag, refname}
-	if err := repo.StrictRun(cmd); err != nil {
+	if _, err := repo.RunQuiet(cmd); err != nil {
 		return err
 	}
 	return nil

branches.go

@@ -0,0 +1,123 @@
+package gitpb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-cmd/cmd"
+)
+
+// returns true if 'git pull' will work
+func (repo *Repo) ExistsUserBranchRemote() bool {
+	branchname := repo.GetUserBranchName()
+	if repo.IsBranchRemote(branchname) {
+		return true
+	}
+	return false
+}
+
+func readRefHash(filename string) string {
+	data, _ := os.ReadFile(filename)
+	return string(data)
+}
+
+func (repo *Repo) GetLocalBranches() []string {
+	return ListFiles(filepath.Join(repo.GetFullPath(), "/.git/refs/heads"))
+}
+
+func (repo *Repo) GetRemoteBranches() []string {
+	remotes := ListFiles(filepath.Join(repo.GetFullPath(), "/.git/refs/remotes"))
+	return remotes
+}
+
+// git describe --tags e548b0fb6d0d14cdfb693850d592419f247dc2b1
+// v0.22.61-15-gbab84d7
+func (repo *Repo) GetHashName(h string) (string, error) {
+	h = strings.TrimSpace(h)
+	// log.Info("GetHashName() is looking for", repo.GetGoPath(), h)
+	cmd := []string{"git", "describe", "--tags", h}
+	r, err := repo.RunQuiet(cmd)
+	if err != nil {
+		return "", err
+	}
+	if len(r.Stdout) == 0 {
+		return "", errors.New("git describe was empty")
+	}
+	return r.Stdout[0], nil
+}
+
+// lookup a hash from a tag with 'git rev-list'
+func (repo *Repo) GetTagHash(t string) string {
+	// git rev-list -n 1 v0.0.66
+	cmd := []string{"git", "rev-list", "-n", "1", t}
+	result, _ := repo.RunStrict(cmd)
+	// log.Info("getLastTagVersion()", result.Stdout)
+
+	if len(result.Stdout) == 0 {
+		// log.Log(WARN, "no gitpb.LastTag() repo is broken. ignore this.", repo.GetGoPath())
+		return ""
+	}
+
+	return result.Stdout[0]
+}
+
+// lookup a hash from a tag with 'git rev-list'
+func (repo *Repo) GetBranchDifferences(to string, from string) []string {
+	// git rev-list -n 1 v0.0.66
+	cmd := []string{"git", "rev-list", to + "..." + from}
+	result, _ := repo.RunStrict(cmd)
+	// log.Info("getLastTagVersion()", result.Stdout)
+
+	// tmp := strings.TrimSpace(strings.Join(result.Stdout, "\n"))
+	// return shell.SplitNewLines(tmp)
+	return result.Stdout
+}
+
+// deletes the devel local branch if it is a subset of the remote devel branch
+func (repo *Repo) DeleteLocalDevelBranch() error {
+	branch := repo.GetDevelBranchName()
+	remote := filepath.Join("origin", branch)
+
+	if !repo.IsDevelRemote() {
+		return fmt.Errorf("no remote branch")
+	}
+
+	b1 := repo.CountDiffObjects(branch, remote) // should be zero
+	if b1 == 0 {
+		cmd := []string{"git", "branch", "-D", repo.GetDevelBranchName()}
+		_, err := repo.RunVerboseOnError(cmd)
+		return err
+	} else {
+		return fmt.Errorf("local branch has patches not in remote")
+	}
+}
+
+// makes a local branch based off of the master branch
+// (unless a remote devel branch exists. then it uses that)
+func (repo *Repo) MakeLocalDevelBranch() (*cmd.Status, error) {
+	branch := repo.GetDevelBranchName()
+	if branch == "" {
+		// hard coded default
+		branch = "devel"
+	}
+
+	if repo.Exists(filepath.Join(".git/refs/heads", branch)) {
+		// local devel branch already exists
+		return nil, nil
+	}
+
+	if repo.Exists(filepath.Join(".git/refs/remotes/origin", branch)) {
+		// remote devel branch exists, but local does not
+		cmd := []string{"git", "checkout", branch}
+		return repo.RunVerboseOnError(cmd)
+	}
+
+	master := repo.GetMasterBranchName()
+	cmd := []string{"git", "branch", branch, master}
+	repo.RunVerboseOnError(cmd)
+	cmd = []string{"git", "checkout", branch}
+	return repo.RunVerboseOnError(cmd)
+}

checkout.go

@@ -1,9 +1,30 @@
 package gitpb
 
-import "go.wit.com/log"
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	"go.wit.com/log"
+)
 
 func (repo *Repo) CheckoutMaster() bool {
 	bName := repo.GetMasterBranchName()
+	if bName == "giterr" {
+		cmd := []string{"git", "checkout", "main"} // todo: figure out main
+		repo.RunVerboseOnError(cmd)
+		os.Exit(-1)
+		// TODO: try to fix this
+		if repo.checkoutBranch("main") {
+			repo.MasterBranchName = "main"
+			return true
+		} else {
+			cmd := []string{"git", "checkout", "main"} // todo: figure out main
+			repo.RunVerboseOnError(cmd)
+			return false
+		}
+	}
 	if repo.checkoutBranch(bName) {
 		return true
 	}
@@ -20,13 +41,22 @@ func (repo *Repo) CheckoutDevel() bool {
 	return false
 }
 
-func (repo *Repo) CheckoutUser() bool {
+func (repo *Repo) CheckoutUser() error {
 	bName := repo.GetUserBranchName()
-	if repo.checkoutBranch(bName) {
-		repo.UserBranchName = bName
-		return true
+	if bName == "uerr" {
+		usr, _ := user.Current()
+		repo.SetUserBranchName(usr.Username)
+		bName = usr.Username
+		log.Info("gitpb CheckoutUser() somehow got user 'uerr'")
 	}
-	return false
+
+	return repo.createUserBranch(bName)
+	/*
+		if err != nil {
+			log.Info("attempting checkout user error", repo.GetGoPath(), bName, err)
+		}
+		return err
+	*/
 }
 
 func (repo *Repo) BranchExists(bName string) bool {
@@ -61,3 +91,58 @@ func (repo *Repo) checkoutBranch(bName string) bool {
 	}
 	return true
 }
+
+// actually creates a local user branch
+func (repo *Repo) createUserBranch(branch string) error {
+	if branch == "" {
+		// get the username here?
+		return fmt.Errorf("gitpb createuserBranch() logic err. git branch name can not be blank")
+	}
+	if repo.IsDirty() {
+		// never change repos on dirty branches
+		return fmt.Errorf("repo is dirty")
+	}
+
+	if repo.Exists(filepath.Join(".git/refs/heads", branch)) {
+		var err error
+		// there is already a local user branch
+		cmd := []string{"git", "checkout", branch}
+		if _, err = repo.RunVerboseOnError(cmd); err == nil {
+			return nil
+		}
+		log.Log(INFO, "git checkout error:", err)
+	}
+
+	if repo.Exists(filepath.Join(".git/refs/remote/origin", branch)) {
+		var err error
+		// there is a remote user branch
+		// todo: check other remotes
+		cmd := []string{"git", "checkout", branch}
+		if _, err = repo.RunVerboseOnError(cmd); err == nil {
+			return nil
+		}
+		log.Log(INFO, "git checkout error:", err)
+	}
+
+	if repo.GetCurrentBranchName() != repo.GetDevelBranchName() {
+		repo.CheckoutDevel()
+	}
+	repo.Reload()
+
+	if repo.GetCurrentBranchName() != repo.GetDevelBranchName() {
+		log.Info("create user branch will probably fail", repo.GetGoPath())
+		// TODO: FIX THIS
+		// return fmt.Errorf("repo must be on devel branch %s", repo.GetGoPath())
+	}
+
+	// create the branch from devel
+	cmd := []string{"git", "branch", branch}
+	if _, err := repo.RunVerboseOnError(cmd); err != nil {
+		return err
+	}
+	cmd = []string{"git", "checkout", branch}
+	if _, err := repo.RunVerboseOnError(cmd); err != nil {
+		return err
+	}
+	return nil
+}

config.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"go.wit.com/lib/protobuf/bugpb"
 	"go.wit.com/log"
 )
 
@@ -26,13 +27,57 @@ func (all *Repos) ConfigSave() error {
 	data, err := all.Marshal()
 	if err != nil {
 		log.Info("gitpb proto.Marshal() failed len", len(data), err)
+		// often this is because strings have invalid UTF-8. This should probably be fixed in the protobuf code
+		if err := all.tryValidate(); err != nil {
+			return err
+		} else {
+			// re-attempt Marshal() here
+			data, err = all.Marshal()
+			if err == nil {
+				// validate & sanitize strings worked
+				configWrite("repos.pb", data)
+				return nil
+			}
+		}
 		return err
 	}
-	log.Info("gitpb.ConfigSave() repos.Marshal() worked len", len(all.Repos), "repos")
 	configWrite("repos.pb", data)
 	return nil
 }
 
+// todo: move this to Marshal() functions automatically in autogenpb?
+func (repo *Repo) ValidateUTF8() error {
+	if _, err := repo.Marshal(); err == nil {
+		// exit if Marshal() works
+		return nil
+	} else {
+		// log.Printf("%s repo.Marshal() failed: %v\n", repo.GetFullPath(), err)
+	}
+	// you only need to do this if Marshal() fails
+	err := bugpb.ValidateProtoUTF8(repo)
+	if err != nil {
+		// log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
+	}
+	if err := bugpb.SanitizeProtoUTF8(repo); err != nil {
+		log.Warn("gitpb.ValidateUTF8()( failed:", err)
+		return err
+	}
+	return nil
+}
+
+func (all *Repos) tryValidate() error {
+	err := bugpb.ValidateProtoUTF8(all)
+	if err != nil {
+		log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
+	}
+	if err := bugpb.SanitizeProtoUTF8(all); err != nil {
+		log.Warn("Sanitation failed:", err)
+		// log.Fatalf("Sanitization failed: %v", err)
+		return err
+	}
+	return nil
+}
+
 // load the repos.pb file. I shouldn't really matter if this
 // fails. the file should be autogenerated. This is used
 // locally just for speed
@@ -56,7 +101,28 @@ func (all *Repos) ConfigLoad() error {
 		all.sampleConfig() // causes nil panic
 		return errors.New("gitpb.ConfigLoad() repos.pb is empty")
 	}
-	return all.Unmarshal(data)
+	err = all.Unmarshal(data)
+	test := NewRepos()
+	if test.Uuid != all.Uuid {
+		log.Log(WARN, "uuids do not match", test.Uuid, all.Uuid)
+		deleteProtobufFile(cfgname)
+	}
+	if test.Version != all.Version {
+		log.Log(WARN, "versions do not match", test.Version, all.Version)
+		deleteProtobufFile(cfgname)
+	}
+	log.Log(INFO, cfgname, "protobuf versions and uuid match", all.Uuid, all.Version)
+	return err
+}
+
+func deleteProtobufFile(filename string) {
+	log.Log(WARN, "The protobuf file format has changed for", filename)
+	log.Log(WARN, "Deleting old file:", filename)
+	log.Log(WARN, "This file will be recreated on the next run.")
+	err := os.Remove(filename)
+	if err != nil {
+		log.Log(WARN, "failed to remove old protobuf file", "err", err)
+	}
 }
 
 func (all *Repos) sampleConfig() {
@@ -83,6 +149,7 @@ func loadFile(fullname string) ([]byte, error) {
 func configWrite(filename string, data []byte) error {
 	fullname := filepath.Join(os.Getenv("FORGE_GOSRC"), filename)
 
+	log.Infof("%s your repos have changed state. cached state. (%d) bytes\n", fullname, len(data))
 	cfgfile, err := os.OpenFile(fullname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	defer cfgfile.Close()
 	if err != nil {

currentVersions.go

@@ -48,35 +48,27 @@ func (repo *Repo) setDevelVersion() {
 	if err == nil {
 		repo.DevelVersion = v
 	} else {
-		log.Log(WARN, "gitpb.GitDevelVersion() error:", err)
-		repo.DevelVersion = "deverr"
+		// log.Log(WARN, "gitpb.GitDevelVersion() error:", err)
+		repo.DevelVersion = ""
 	}
 }
 
 func (repo *Repo) setUserVersion() {
 	bname := repo.GetUserBranchName()
+	if !repo.Exists(filepath.Join(".git/refs/heads", bname)) {
+		// the user branch does not exist at this time
+		repo.UserVersion = "uerr"
+		return
+	}
 	v, err := repo.gitVersionByName(bname)
 	if err == nil {
 		repo.UserVersion = v
 	} else {
-		log.Log(WARN, "gitpb.GitUserVersion() error:", err)
+		// log.Log(WARN, "gitpb.GitUserVersion() error:", err)
 		repo.UserVersion = "uerr"
 	}
 }
 
-/*
-now tracked in repo.Reload()
-func (repo *Repo) GetCurrentBranchName() string {
-	r := repo.RunQuiet([]string{"git", "branch", "--show-current"})
-	output := strings.Join(r.Stdout, "\n")
-	if r.Error != nil {
-		log.Log(WARN, "GetCurrentBranchName() not in a git repo?", r.Error, repo.GetGoPath())
-		log.Log(WARN, "GetCurrentBranchName() output might have worked anyway:", output)
-	}
-	return strings.TrimSpace(output)
-}
-*/
-
 // this is used often. probably move everything to this
 // returns things like
 // v0.2.2
@@ -98,13 +90,13 @@ func (repo *Repo) gitDescribeByHash(hash string) (string, error) {
 	if hash == "" {
 		return "", errors.New("hash was blank")
 	}
-	r := repo.RunQuiet([]string{"git", "describe", "--tags", "--always", hash})
+	r, err := repo.RunQuiet([]string{"git", "describe", "--tags", "--always", hash})
 	out := strings.Join(r.Stdout, "\n")
-	if r.Error != nil {
-		log.Warn("not in a git repo or bad hash?", r.Error, repo.GetGoPath())
-		return out, r.Error
+	if err != nil {
+		log.Warn("not in a git repo or bad hash?", err, repo.GetGoPath())
+		return out, err
 	}
-	return out, r.Error
+	return out, err
 }
 
 // this should get the most recent tag
@@ -142,26 +134,26 @@ func (repo *Repo) gitVersionByName(name string) (string, error) {
 
 	if name == "" {
 		// git will return the current tag
-		r := repo.RunQuiet([]string{"git", "describe", "--tags", "--always"})
+		r, err := repo.RunQuiet([]string{"git", "describe", "--tags", "--always"})
 		output := strings.Join(r.Stdout, "\n")
-		if r.Error != nil {
+		if err != nil {
 			log.Log(WARN, "gitDescribeByName() output might have worked anyway:", output)
-			log.Log(WARN, "gitDescribeByName() not in a git repo?", r.Error, repo.GetGoPath())
-			return "", r.Error
+			log.Log(WARN, "gitDescribeByName() not in a git repo?", err, repo.GetGoPath())
+			return "", err
 		}
 		return strings.TrimSpace(output), nil
 	}
 	if !repo.IsBranch(name) {
 		// tag does not exist
-		log.Log(WARN, "LocalTagExists()", name, "did not exist")
+		// log.Log(WARN, "LocalTagExists()", name, "did not exist")
 		return "", errors.New("gitDescribeByName() git fatal: Not a valid object name: " + name)
 	}
 	cmd := []string{"git", "describe", "--tags", "--always", name}
-	result := repo.RunQuiet(cmd)
+	result, err := repo.RunQuiet(cmd)
 	output := strings.Join(result.Stdout, "\n")
-	if result.Error != nil {
+	if err != nil {
 		log.Log(WARN, "cmd =", cmd)
-		log.Log(WARN, "err =", result.Error)
+		log.Log(WARN, "err =", err)
 		log.Log(WARN, "output (might have worked with error?) =", output)
 		log.Log(WARN, "not in a git repo or bad tag?", repo.GetGoPath())
 		return "", result.Error
@@ -170,56 +162,6 @@ func (repo *Repo) gitVersionByName(name string) (string, error) {
 	return strings.TrimSpace(output), nil
 }
 
-// find a branch name
-// will find "master" or "devel"
-// will also find "v0.1.1"
-// or will find "patches-from-foo"
-// will return *any* match on any git branch because it doesn't
-// matter much here yet
-// eventually this will be worked out by forge in some future code that hasn't been made yet
-func (repo *Repo) IsBranch(findname string) bool {
-	loop := repo.Tags.All()
-	for loop.Scan() {
-		t := loop.Next()
-		// log.Info("LocalTagExists() tag:", t.Refname)
-
-		tagname := t.Refname
-		if strings.HasPrefix(tagname, "refs/remotes") {
-			continue
-		}
-		path, filename := filepath.Split(tagname)
-		log.Log(INFO, "gitpb.IsBranch() tag:", path, filename, "from", repo.GetGoPath())
-		if filename == findname {
-			log.Log(INFO, "gitpb.IsBranch() found tag:", path, filename, "from", repo.GetGoPath())
-			return true
-		}
-	}
-	log.Log(INFO, "did not find tag:", findname, "in", repo.GetGoPath())
-	return false
-}
-
-// todo: redo this and above. both are messed up. ignore for now until things are stable
-func (repo *Repo) IsLocalBranch(findname string) bool {
-	loop := repo.Tags.All()
-	for loop.Scan() {
-		t := loop.Next()
-		// log.Info("LocalTagExists() tag:", t.Refname)
-
-		tagname := t.Refname
-		if strings.HasPrefix(tagname, "refs/heads") {
-			continue
-		}
-		path, filename := filepath.Split(tagname)
-		log.Log(INFO, "gitpb.IsBranch() tag:", path, filename, "from", repo.GetGoPath())
-		if filename == findname {
-			log.Log(INFO, "gitpb.IsBranch() found tag:", path, filename, "from", repo.GetGoPath())
-			return true
-		}
-	}
-	log.Log(INFO, "did not find tag:", findname, "in", repo.GetGoPath())
-	return false
-}
-
 func trimNonNumericFromStart(s string) string {
 	for i, r := range s {
 		if unicode.IsDigit(r) {
@@ -289,23 +231,25 @@ func (repo *Repo) IncrementTargetMinor() {
 }
 
 // changes the target revision. v0.1.3 becomes v0.1.4
-func (repo *Repo) IncrementTargetRevision() bool {
+func (repo *Repo) IncrementTargetRevision() {
 	// first try just going from the last tag
 	repo.incrementRevision(repo.GetLastTag())
 
 	if !isNewerVersion(repo.GetMasterVersion(), repo.GetTargetVersion()) {
-		log.Printf("master version() %s is higher than target version %s\n", repo.GetMasterVersion(), repo.GetTargetVersion())
+		// log.Printf("tag error. master version() %s was higher than target version %s\n", repo.GetMasterVersion(), repo.GetTargetVersion())
 		repo.incrementRevision(repo.GetMasterVersion())
 	}
-	if !isNewerVersion(repo.GetLastTag(), repo.GetTargetVersion()) {
-		log.Printf("last tag versn() %s is higher than target version %s\n", repo.GetLastTag(), repo.GetTargetVersion())
-		return false
-	}
-	if !isNewerVersion(repo.GetMasterVersion(), repo.GetTargetVersion()) {
-		log.Printf("master version() %s is higher than target version %s\n", repo.GetMasterVersion(), repo.GetTargetVersion())
-		return false
-	}
-	return true
+	/*
+		if !isNewerVersion(repo.GetLastTag(), repo.GetTargetVersion()) {
+			log.Printf("last tag versn() %s is higher than target version %s\n", repo.GetLastTag(), repo.GetTargetVersion())
+			return false
+		}
+		if !isNewerVersion(repo.GetMasterVersion(), repo.GetTargetVersion()) {
+			log.Printf("master version() %s is higher than target version %s\n", repo.GetMasterVersion(), repo.GetTargetVersion())
+			return false
+		}
+		return true
+	*/
 }
 
 func (repo *Repo) incrementRevision(lasttag string) {

gitTag.byAge.go

@@ -12,7 +12,7 @@ import (
 // todo: probably switch to using slices. new things added in 1.23
 // https://pkg.go.dev/slices
 
-func (all *GitTags) newSort() *GitTagIterator {
+func (all *GitTags) newSort() *GitTagScanner {
 	slices.SortFunc(all.GitTags, func(a, b *GitTag) int {
 		if n := strings.Compare(a.Name, b.Name); n != 0 {
 			return n
@@ -44,12 +44,12 @@ func (all *GitTags) GetAge(name string) time.Time {
 	return newest
 }
 
-func (all *GitTags) SortByAge() *GitTagIterator {
+func (all *GitTags) SortByAge() *GitTagScanner {
 	packs := all.selectAllGitTags()
 
 	sort.Sort(GitTagAge(packs))
 
-	iterator := newGitTagIterator(packs)
+	iterator := newGitTagScanner(packs)
 	return iterator
 }
 
@@ -87,6 +87,25 @@ func (repo *Repo) NewestAge() time.Duration {
 	return time.Since(newest)
 }
 
+func (repo *Repo) NewestTime() time.Time {
+	alltags := repo.Tags.selectAllGitTags()
+
+	var newest time.Time
+
+	for _, tag := range alltags {
+		// check the actual age of the patch
+		if newest.Before(tag.Authordate.AsTime()) {
+			newest = tag.Authordate.AsTime()
+		}
+		// check the age of the commit
+		if newest.Before(tag.Creatordate.AsTime()) {
+			newest = tag.Creatordate.AsTime()
+		}
+	}
+
+	return newest
+}
+
 func (repo *Repo) NewestAgeVerbose() time.Duration {
 	alltags := repo.Tags.selectAllGitTags()
 
@@ -104,3 +123,34 @@ func (repo *Repo) NewestAgeVerbose() time.Duration {
 
 	return time.Since(newest)
 }
+
+// not really accurate. temprorary until git Config() parsing is better
+func (repo *Repo) BranchAge(branch string) time.Duration {
+	alltags := repo.Tags.selectAllGitTags()
+
+	var newest time.Time
+	var cur time.Time
+	var auth time.Time
+
+	for _, tag := range alltags {
+		cur = tag.Creatordate.AsTime()
+		auth = tag.Authordate.AsTime()
+		if branch == filepath.Base(tag.Refname) {
+			// log.Info("\t\tfound tag", i, branch, tag.Authordate.AsTime(), tag.Creatordate.AsTime())
+			if cur.Before(auth) {
+				return time.Since(auth)
+			}
+			return time.Since(cur)
+		}
+
+		// check both dates I guess
+		if newest.Before(auth) {
+			newest = auth
+		}
+		if newest.Before(cur) {
+			newest = cur
+		}
+	}
+
+	return time.Since(newest)
+}

gitTag.common.go

@@ -1,11 +1,16 @@
 package gitpb
 
-func (repo *Repo) DevelHash() string {
+import (
+	"path/filepath"
+	"strings"
+
+	"go.wit.com/log"
+)
+
+func (repo *Repo) ActualDevelHash() string {
 	brname := repo.GetDevelBranchName()
 	refname := "refs/heads/" + brname
-	all := repo.Tags.All()
-	for all.Scan() {
-		tag := all.Next()
+	for tag := range repo.Tags.IterAll() {
 		// log.Info("repo tag", tag.GetHash(), tag.GetRefname())
 		if tag.GetRefname() == refname {
 			return tag.GetHash()
@@ -13,3 +18,128 @@ func (repo *Repo) DevelHash() string {
 	}
 	return ""
 }
+
+func (repo *Repo) GetLocalHash(brname string) string {
+	refname := "refs/heads/" + brname
+	for tag := range repo.Tags.IterAll() {
+		// log.Info("repo tag", tag.GetHash(), tag.GetRefname())
+		if tag.GetRefname() == refname {
+			return strings.TrimSpace(tag.GetHash())
+		}
+	}
+	return ""
+}
+
+func (repo *Repo) GetRemoteHash(brname string) string {
+	refname := "refs/remotes/origin/" + brname
+	for tag := range repo.Tags.IterAll() {
+		// log.Info("repo tag", tag.GetHash(), tag.GetRefname())
+		if tag.GetRefname() == refname {
+			return strings.TrimSpace(tag.GetHash())
+		}
+	}
+	return ""
+}
+
+// this is the correct way. uses 'git show-ref'
+func (repo *Repo) IsBranchRemote(brname string) bool {
+	if repo.Tags == nil {
+		return false
+	}
+
+	brname = "refs/remotes/origin/" + brname
+	ref := repo.Tags.FindByRefname(brname)
+	if ref == nil {
+		// log.Info("did not found refname!!!!!!!!", brname)
+		return false
+	}
+	// log.Info("found refname!!!!!!!!")
+	return true
+}
+
+// this is the correct way. uses 'git show-ref'
+func (repo *Repo) IsDevelRemote() bool {
+	if repo.Tags == nil {
+		return false
+	}
+
+	devname := repo.GetDevelBranchName()
+	refname := "refs/remotes/origin/" + devname
+	ref := repo.Tags.FindByRefname(refname)
+	if ref == nil {
+		// log.Info("did not found refname!!!!!!!!", refname)
+		return false
+	}
+	// log.Info("found refname!!!!!!!!")
+	return true
+}
+
+// find a branch namm
+// will find "master" or "devel"
+// will also find "v0.1.1"
+// or will find "patches-from-foo"
+// will return *any* match on any git branch because it doesn't
+// matter much here yet
+// eventually this will be worked out by forge in some future code that hasn't been made yet
+func (repo *Repo) IsBranch(findname string) bool {
+	for t := range repo.Tags.IterAll() {
+		// log.Info("LocalTagExists() tag:", t.Refname)
+
+		tagname := t.Refname
+		if strings.HasPrefix(tagname, "refs/remotes") {
+			continue
+		}
+		path, filename := filepath.Split(tagname)
+		log.Log(INFO, "gitpb.IsBranch() tag:", path, filename, "from", repo.GetGoPath())
+		if filename == findname {
+			log.Log(INFO, "gitpb.IsBranch() found tag:", path, filename, "from", repo.GetGoPath())
+			return true
+		}
+	}
+	log.Log(INFO, "did not find tag:", findname, "in", repo.GetGoPath())
+	return false
+}
+
+func (repo *Repo) IsLocalBranch(findname string) bool {
+	for t := range repo.Tags.IterAll() {
+		if !strings.HasPrefix(t.Refname, "refs/heads") {
+			// log.Info("LocalTagExists() skip tag:", t.Refname)
+			continue
+		}
+		// log.Info("LocalTagExists() check tag:", t.Refname)
+		path, filename := filepath.Split(t.Refname)
+		log.Log(INFO, "gitpb.IsBranch() tag:", path, filename, "from", repo.GetGoPath())
+		if filename == findname {
+			log.Log(INFO, "gitpb.IsBranch() found tag:", path, filename, "from", repo.GetGoPath())
+			return true
+		}
+	}
+	log.Log(INFO, "did not find tag:", findname, "in", repo.GetGoPath())
+	return false
+}
+
+// finds the newest tag. used for deciding if master needs to be published
+func (repo *Repo) FindLastTag() string {
+	var newest *GitTag
+	for tag := range repo.Tags.IterAll() {
+		if !strings.HasPrefix(tag.GetRefname(), "refs/tags/") {
+			continue
+		}
+		if newest == nil {
+			newest = tag
+			continue
+		}
+		cur := newest.Creatordate.AsTime()
+		if cur.Before(tag.Creatordate.AsTime()) {
+			newest = tag
+		}
+		// newtag := strings.TrimPrefix(tag.GetRefname(), "refs/tags/")
+		// log.Info("repo tag", tag.GetHash(), tag.Creatordate.AsTime(), tag.GetRefname(), newtag)
+	}
+	if newest == nil {
+		return ""
+	}
+	// log.Info("repo newest tag", newest.GetHash(), newest.Creatordate.AsTime(), newest.GetRefname())
+	newtag := strings.TrimPrefix(newest.GetRefname(), "refs/tags/")
+	return newtag
+}

gitTag.proto

@@ -4,17 +4,38 @@ package gitpb;
 
 import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp
 
-
-message GitTag {					// `autogenpb:nomutex`
-	string	refname 			= 1;	// `autogenpb:unique` `autogenpb:sort` 	// tag name. treated as unique
-	google.protobuf.Timestamp creatordate	= 2;	// git creatordate
-	google.protobuf.Timestamp authordate	= 3;	// git author date
-	string	hash				= 4;	// `autogenpb:unique` // git hash
-	string	subject				= 5;	// git tag subject
+message GitRemote {                                       // `autogenpb:nomutex`
+        string                      url             = 1;
+        string                      fetch           = 2;
 }
 
-message GitTags {			// `autogenpb:marshal` `autogenpb:nomutex`
-	string   uuid	        = 1;    // `autogenpb:uuid:ffdff813-0316-4372-9e82-4c1c7d202526`
-	string   version        = 2;    // `autogenpb:version:v0.0.47`
-	repeated GitTag gitTags	= 3;
+message GitBranch {                                       // `autogenpb:nomutex`
+        string                      remote          = 1;  // the name of the remote repo
+        string                      merge           = 2;  // the merge path from the config file
+        string                      name            = 3;  // the branch name from the config file
+}
+
+// readGitConfig reads and parses the .git/config file
+message GitConfig {                                       // `autogenpb:nomutex`
+        map<string, string>         core            = 1;  // map[origin] = "https:/git.wit.org/gui/gadgets"
+        map<string, GitRemote>      remotes         = 2;  // map[origin] = "https:/git.wit.org/gui/gadgets"
+        map<string, GitBranch>      branches        = 3;  // map[guimaster] = origin guimaster
+        map<string, string>         submodules      = 4;
+        map<string, string>         hashes          = 5;
+        map<string, string>         versions        = 6;
+        repeated GitBranch          local           = 7;  // move this this and away from the map<> variables
+}
+
+message GitTag {                                          // `autogenpb:nomutex`
+        string                      refname         = 1;  // `autogenpb:unique` `autogenpb:sort` 	// tag name. treated as unique
+        google.protobuf.Timestamp   creatordate     = 2;  // git creatordate
+        google.protobuf.Timestamp   authordate      = 3;  // git author date
+        string                      hash            = 4;  // `autogenpb:unique` // git hash
+        string                      subject         = 5;  // git tag subject
+}
+
+message GitTags {                                         // `autogenpb:marshal` `autogenpb:nomutex`
+        string                      uuid            = 1;  // `autogenpb:uuid:ffdff813-0316-4372-9e82-4c1c7d202526`
+        string                      version         = 2;  // `autogenpb:version:v0.0.47`
+        repeated GitTag             gitTags         = 3;
 }

goDep.parseGoSum.go

@@ -4,15 +4,18 @@ package gitpb
 
 import (
 	"bufio"
+	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"go.wit.com/lib/gui/shell"
 	"go.wit.com/log"
 )
 
-// reads and parses the go.sum file
-// does not change anything
+// reads and parses the go.sum file into a protobuf struct
+// this function isn't supposed to change anything, just parse the existing files
 func (repo *Repo) ParseGoSum() bool {
 	// empty out what was there before
 	repo.GoDeps = nil
@@ -21,6 +24,7 @@ func (repo *Repo) ParseGoSum() bool {
 	// that means, there is not a go.sum file
 	// because the package is completely self contained!
 	if err := repo.setPrimitive(); err != nil {
+		// temporarily enabled this. this is really noisy
 		// log.Info("gitpb.ParseGoSum()", err)
 		return false
 	}
@@ -68,3 +72,127 @@ func (repo *Repo) ParseGoSum() bool {
 	}
 	return true
 }
+
+// attempt to parse go.* files in a directory
+func GoSumParseDir(moddir string) (*GoDeps, error) {
+	isprim, err := computePrimitive(moddir)
+	if err != nil {
+		// "go mod init" failed
+		return nil, err
+	}
+	if isprim {
+		// might be a GO primitive. no go.sum file
+		return nil, nil
+	}
+	// go.sum exists. parse the go.sum file
+	return parseGoSum(moddir)
+}
+
+// Detect a 'Primitive' package. Sets the isPrimitive flag
+// will return true if the repo is truly not dependent on _anything_ else
+// like spew or lib/widget
+// it assumes 'go mod init' and 'go mod tidy' ran without error
+func computePrimitive(moddir string) (bool, error) {
+	// go mod init & go mod tidy ran without errors
+	log.Log(INFO, "isPrimitiveGoMod()", moddir)
+	gomod, err := os.Open(filepath.Join(moddir, "go.mod"))
+	if err != nil {
+		log.Log(INFO, "missing go.mod", moddir)
+		return false, err
+	}
+	defer gomod.Close()
+
+	if shell.Exists(filepath.Join(moddir, "go.sum")) {
+		return false, nil
+	}
+
+	scanner := bufio.NewScanner(gomod)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+
+		parts := strings.Fields(line)
+		log.Log(INFO, "  gomod:", parts)
+		if len(parts) >= 1 {
+			log.Log(INFO, "  gomod: part[0] =", parts[0])
+			if parts[0] == "require" {
+				log.Log(INFO, "  should return false here")
+				return false, errors.New("go.mod file is not primitive")
+			}
+			/*
+				if parts[0] == "go" {
+					if parts[1] != "1.21" {
+						log.Log(WARN, "go not set to 1.21 for", repo.GetGoPath())
+						// return false, errors.New("go not set to 1.21 for " + repo.GetGoPath())
+					}
+				}
+			*/
+		}
+	}
+	return true, nil
+}
+
+// parse the go.sum file into a protobuf
+func parseGoSum(moddir string) (*GoDeps, error) {
+	godeps := new(GoDeps)
+
+	tmp, err := os.Open(filepath.Join(moddir, "go.sum"))
+	defer tmp.Close()
+	if err != nil {
+		log.Info("gitpb.ParseGoSum() missing go.sum. Some error happened with go mod init & tidy", err)
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(tmp)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+
+		parts := strings.Split(line, " ")
+		if len(parts) == 3 {
+			godep := strings.TrimSpace(parts[0])
+			version := strings.TrimSpace(parts[1])
+			if strings.HasSuffix(version, "/go.mod") {
+				version = strings.TrimSuffix(version, "/go.mod")
+			}
+			new1 := GoDep{
+				GoPath:  godep,
+				Version: version,
+			}
+			godeps.AppendByGoPath(&new1)
+		} else {
+			return nil, fmt.Errorf("gitpb.ParseGoSum() go.sum parse error invalid: %s", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		godeps = nil
+		return nil, err
+	}
+	return godeps, nil
+}
+
+func (repo *Repo) GoSumFromPkgDir() (*GoDeps, error) {
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		return nil, err
+	}
+	rver := repo.GetLastTag()
+	if rver == "" {
+		return nil, errors.New("could not get master version")
+	}
+	goget := repo.GetGoPath() + "@" + rver
+	moddir := filepath.Join(homedir, "go/pkg/mod", repo.GetGoPath()+"@"+rver)
+
+	if !shell.IsDir(moddir) {
+		cmd := []string{"go", "get", goget}
+		repo.RunVerboseOnError(cmd)
+	}
+
+	if !shell.IsDir(moddir) {
+		return nil, errors.New("missing go/pkg/mod. Run: go get " + goget)
+	}
+	return GoSumParseDir(moddir)
+}
+
+func (repo *Repo) GoSumFromRepo() (*GoDeps, error) {
+	return GoSumParseDir(repo.GetFullPath())
+}

goDep.proto

@@ -8,16 +8,16 @@ import "google/protobuf/timestamp.proto"; // Import the well-known type for Time
 
 // global settings for autogenpb `autogenpb:mutex`
 
-message GoDep {						// `autogenpb:nomutex`
-	string	hash 				= 1;	// `autogenpb:unique` `autogenpb:sort`	// md5sum/hash value from the go.sum file
-	google.protobuf.Timestamp ctime		= 2;	// get the go date from 'go list' ?
-	string	version 			= 3;	// v1.2.2
-	string	goPath 				= 4;	// `autogenpb:unique`  `autogenpb:sort` // "go.wit.com/lib/foo"
-	string	goVersion			= 5;	// version of golang the developer used to make this package version
+message GoDep {                                         // `autogenpb:nomutex`
+        string                      hash          = 1;  // `autogenpb:unique` `autogenpb:sort`	// md5sum/hash value from the go.sum file
+        google.protobuf.Timestamp   ctime         = 2;  // get the go date from 'go list' ?
+        string                      version       = 3;  // v1.2.2
+        string                      goPath        = 4;  // `autogenpb:unique`  `autogenpb:sort` // "go.wit.com/lib/foo"
+        string                      goVersion     = 5;  // version of golang the developer used to make this package version
 }
 
-message GoDeps {			// `autogenpb:nomutex`
-	string   uuid	        = 1;    // `autogenpb:uuid:7de62c09-b335-4d80-902d-08552c501b7c`
-	string   version        = 2;    // `autogenpb:version:v0.0.51`
-	repeated GoDep goDeps	= 3;	// `autogenpb:unique`  `autogenpb:sort`
+message GoDeps {                                        // `autogenpb:nomutex`
+        string                      uuid          = 1;  // `autogenpb:uuid:7de62c09-b335-4d80-902d-08552c501b7c`
+        string                      version       = 2;  // `autogenpb:version:v0.0.51`
+        repeated GoDep              goDeps        = 3;  // `autogenpb:unique`  `autogenpb:sort`
 }

http.go

@@ -0,0 +1,81 @@
+// Copyright 1994-2025 WIT.COM Inc Licensed GPL 3.0
+
+package gitpb
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/user"
+
+	"go.wit.com/log"
+)
+
+func httpPost(url string, data []byte) ([]byte, error) {
+	var err error
+	var req *http.Request
+
+	req, err = http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
+	log.Info("httpPost() with len", len(data), "url", url)
+	if err != nil {
+		log.Error(err)
+		return nil, err
+	}
+
+	usr, _ := user.Current()
+	req.Header.Set("author", usr.Username)
+	req.Header.Set("hostname", "fixme:hostname")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return []byte("client.Do(req) error"), err
+	}
+	defer resp.Body.Close()
+	// log.Info("httpPost() with len", len(data))
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Error(err)
+		return body, err
+	}
+
+	log.Info("gitpb.httpPost() worked", url)
+	return body, nil
+}
+
+func (r *Repos) SubmitReposPB(url string) (*Repos, error) {
+	msg, err := r.Marshal()
+	if err != nil {
+		log.Info("Marshal() failed:", err)
+		return nil, err
+	}
+	log.Info("proto.Marshal() msg len", len(msg))
+	body, err := httpPost(url, msg)
+	if err != nil {
+		log.Info("httpPost() failed:", err)
+		return nil, err
+	}
+	log.Info("httpPost() worked with url:", url, "len(body) =", len(body))
+	if err := r.Unmarshal(body); err != nil {
+		log.Info("SubmitReposPB() Unmarshal() failed:", err)
+		log.Printf("%s\n", body)
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func (r *Repos) SendPB(w http.ResponseWriter) error {
+	data, err := r.Marshal()
+	if err != nil {
+		log.Info("Marshal() failed:", err)
+		return err
+	}
+	log.Info("SendPB() Marshal() len(data)", len(data))
+	fmt.Fprintf(w, "%s", data)
+
+	return nil
+}

makePatches.go

@@ -1,112 +0,0 @@
-package gitpb
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-
-	"go.wit.com/log"
-)
-
-type Patch struct {
-	GoPath  string
-	Ref     string
-	giturl  string
-	comment string
-}
-
-// move all this to repolist and gowit repos
-
-func (repo *Repo) GetPatches(oldname string, newname string) (int, []*Patch) {
-	var patchcount int
-	patches := make([]*Patch, 0, 0)
-
-	if oldname == newname {
-		return 0, nil
-	}
-	// log.Info("repo userv, develv", userv, develv)
-	gitcmd := []string{"git", "log", "--oneline", oldname + ".." + newname}
-	log.Info("Run:", gitcmd)
-	r := repo.Run(gitcmd)
-	if r.Error != nil {
-		log.Info("git failed ", repo.GetGoPath(), "err =", r.Error)
-		return 0, nil
-	}
-
-	// patches = strings.Split(output, "\n")
-	log.Info("Run:", r.Stdout)
-	for _, line := range r.Stdout {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.Split(line, " ")
-		newp := new(Patch)
-		newp.Ref = parts[0]
-		newp.comment = strings.Join(parts[1:], " ")
-		log.Info("Patch line:", line, repo.GetGoPath())
-		patchcount += 1
-		patches = append(patches, newp)
-	}
-	return patchcount, patches
-}
-
-func (repo *Repo) GetUserPatches() (int, []*Patch) {
-	usern := repo.GetUserBranchName()
-	develn := repo.GetDevelBranchName()
-	userv := repo.GetUserVersion()
-	develv := repo.GetDevelVersion()
-
-	if userv == develv {
-		return 0, nil
-	}
-
-	c, all := repo.GetPatches(develn, usern)
-	log.Info("GetPatches() guireleaser", develn, usern, "count =", c)
-	return c, all
-}
-
-func (repo *Repo) GetMasterPatches() (int, []*Patch) {
-	lasttag := repo.GetLastTag()
-	mastern := repo.GetMasterBranchName()
-	masterv := repo.GetMasterVersion()
-
-	if lasttag == masterv {
-		return 0, nil
-	}
-
-	c, all := repo.GetPatches(lasttag, mastern)
-	log.Info("GetPatches() guireleaser", lasttag, mastern, "count =", c)
-	return c, all
-}
-
-func (all *Repos) MakePatchset(setdir string) bool {
-	loop := all.SortByFullPath()
-	for loop.Scan() {
-		repo := loop.Next()
-		log.Info("repo", repo.GetGoPath())
-		userv := repo.GetUserVersion()
-		develv := repo.GetDevelVersion()
-		usern := repo.GetUserBranchName()
-		develn := repo.GetDevelBranchName()
-		if userv == develv {
-			// this repo is unchanged
-			continue
-		}
-
-		repodir := filepath.Join(setdir, repo.GetGoPath())
-		os.MkdirAll(repodir, os.ModeDir)
-		// git format-patch branch1..branch2
-		gitcmd := []string{"git", "format-patch", "-o", repodir, develn + ".." + usern}
-		log.Info("Run:", gitcmd)
-		r := repo.Run(gitcmd)
-		log.Info("output =", r.Stdout)
-		if r.Error == nil {
-			log.Info("patches made okay for:", repo.GetGoPath())
-			continue
-		}
-		log.Info("patches failed for:", repo.GetGoPath())
-		return false
-	}
-	return true
-}

reload.go

@@ -8,8 +8,28 @@ import (
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 )
 
-// TODO: fix and clean this up. this is a work in progress
+// does a fast check with os.Stat()
+// if the mtimes changed, does a full repo.Reload()
+func (repo *Repo) ReloadCheck() error {
+	if !repo.DidRepoChange() {
+		return nil
+	}
+	// f.configSave = true
+	err := repo.Reload()
+	return err
+}
+
+// TODO: clean this up more, but it is working now more or less
 func (repo *Repo) Reload() error {
+	// sometimes, on new repos, if .git/HEAD does not exist
+	// defective git daemons or badly configured repos, 'git clone' can fail
+	// if so, 'git fetch origin' can repair the state
+	if !repo.Exists(".git/HEAD") {
+		cmd := []string{"git", "fetch", "origin"}
+		repo.RunVerbose(cmd)
+		cmd = []string{"git", "checkout", "main"} // todo: figure out main
+		repo.RunVerbose(cmd)
+	}
 	// log.Info("in reload", repo.FullPath)
 	repo.Tags = new(GitTags)
 	repo.reloadGitTags()
@@ -28,8 +48,15 @@ func (repo *Repo) Reload() error {
 	repo.CheckDirty()
 	repo.setRepoState()
 
+	if repo.GitConfig == nil {
+		if err := repo.updateGitConfig(); err != nil {
+			return err
+		}
+	}
+
 	// LastUpdate should always be the newest time
 	repo.Times.LastUpdate = timestamppb.New(time.Now())
+	repo.ValidateUTF8()
 	return nil
 }
 
@@ -41,38 +68,17 @@ func (repo *Repo) SetUserBranchName(bname string) {
 	repo.UserBranchName = bname
 }
 
-// updates LastTag // todo, get this from the protobuf
+// updates LastTag by age
 func (repo *Repo) setLastTag() {
-	cmd := []string{"git", "rev-list", "--tags", "--max-count=1"}
-	result := repo.RunQuiet(cmd)
-	// log.Info("getLastTagVersion()", result.Stdout)
-
-	if len(result.Stdout) != 1 {
-		log.Log(WARN, "git LastTag() error:", result.Stdout)
-		repo.LastTag = ""
-		return
-	}
-
-	hash := result.Stdout[0]
-
-	cmd = []string{"git", "describe", "--tags", "--always", hash}
-	result = repo.RunQuiet(cmd)
-
-	if len(result.Stdout) != 1 {
-		log.Log(WARN, "git LastTag() error:", result.Stdout)
-		repo.LastTag = ""
-		return
-	}
-
-	repo.LastTag = result.Stdout[0]
+	repo.LastTag = repo.FindLastTag()
 }
 
 func (repo *Repo) setCurrentBranchName() {
 	repo.CurrentBranchName = ""
-	r := repo.RunQuiet([]string{"git", "branch", "--show-current"})
+	r, err := repo.RunQuiet([]string{"git", "branch", "--show-current"})
 	output := strings.Join(r.Stdout, "\n")
-	if r.Error != nil {
-		log.Log(WARN, "GetCurrentBranchName() not in a git repo?", r.Error, repo.GetGoPath())
+	if err != nil {
+		log.Log(WARN, "GetCurrentBranchName() not in a git repo?", err, repo.GetGoPath())
 		log.Log(WARN, "GetCurrentBranchName() output might have worked anyway:", output)
 	}
 	repo.CurrentBranchName = strings.TrimSpace(output)
@@ -85,10 +91,10 @@ func (repo *Repo) setCurrentBranchVersion() {
 		log.Info("repo.GetCurrentBranchVersion() repo == nil")
 		return
 	}
-	r := repo.RunQuiet([]string{"git", "describe", "--tags", "--always"})
+	r, err := repo.RunQuiet([]string{"git", "describe", "--tags", "--always"})
 	output := strings.Join(r.Stdout, "\n")
-	if r.Error != nil {
-		log.Log(WARN, "GetCurrentBranchVersion() not in a git repo?", r.Error, repo.GetGoPath())
+	if err != nil {
+		log.Log(WARN, "GetCurrentBranchVersion() not in a git repo?", err, repo.GetGoPath())
 		log.Log(WARN, "GetCurrentBranchVersion() output might have worked anyway:", output)
 	}
 	repo.CurrentBranchVersion = strings.TrimSpace(output)

reloadBranches.go

@@ -1,6 +1,7 @@
 package gitpb
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -62,7 +63,7 @@ func (repo *Repo) CheckBranches() bool {
 		if hash == hashCheck {
 			// log.Info("notsure why this git show is here", hash)
 		} else {
-			log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
+			// log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
 			perfect = false
 		}
 	}
@@ -113,6 +114,22 @@ func ListFiles(directory string) []string {
 	return files
 }
 
+// forge doesn't want a remote user branch
+// this will make sure the user branch is only local
+func (repo *Repo) checkUserBranch() error {
+	ubn := repo.GetUserBranchName()
+	log.Info("user branch name:", ubn, repo.GetGoPath())
+	if repo.GitConfig == nil {
+		return fmt.Errorf("GitConfig == nil")
+	}
+
+	for _, l := range repo.GitConfig.Local {
+		log.Info("local branch name:", l.Name)
+	}
+
+	return nil
+}
+
 func (repo *Repo) ExamineBranches() *GitTag {
 	var hashCheck string
 	all := repo.GetBranches()
@@ -153,7 +170,7 @@ func (repo *Repo) ExamineBranches() *GitTag {
 		if hash == hashCheck {
 			// log.Info("notsure why this git show is here", hash)
 		} else {
-			log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
+			// log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
 			tag := new(GitTag)
 			tag.Refname = b
 			tag.Hash = hash

reloadCheckDirty.go

@@ -62,5 +62,4 @@ func (repo *Repo) CheckDirty() bool {
 		repo.State = "dirty"
 	}
 	return bad
-
 }

reloadGitConfig.go

@@ -0,0 +1,196 @@
+package gitpb
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go.wit.com/log"
+)
+
+// does processing on the go.mod and go.sum files
+
+func (repo *Repo) updateGitConfig() error {
+	if repo == nil {
+		return fmt.Errorf("gitpb.updateGitConfig() repo == nil")
+	}
+	if repo.GitConfig == nil {
+		repo.GitConfig = new(GitConfig)
+	}
+
+	repo.GitConfig.Core = make(map[string]string)
+	repo.GitConfig.Remotes = make(map[string]*GitRemote)
+	repo.GitConfig.Branches = make(map[string]*GitBranch)
+	repo.GitConfig.Submodules = make(map[string]string)
+	repo.GitConfig.Versions = make(map[string]string)
+	repo.GitConfig.Hashes = make(map[string]string)
+	return repo.readGitConfig()
+}
+
+// readGitConfig reads and parses the .git/config file
+func (repo *Repo) readGitConfig() error {
+	filename := filepath.Join(repo.GetFullPath(), ".git/config")
+	file, err := os.Open(filename)
+	defer file.Close()
+	if err != nil {
+		return err
+	}
+
+	var currentSection string = ""
+	var currentName string = ""
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+
+		// Skip empty lines and comments
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
+			continue
+		}
+
+		// Check for section headers
+		if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
+			line = strings.Trim(line, "[]")
+			parts := strings.Split(line, " ")
+			currentSection = parts[0]
+
+			if len(parts) == 2 {
+				line = strings.Trim(line, "[]")
+				currentName = strings.Trim(parts[1], "\"")
+			}
+			continue
+		}
+
+		partsNew := strings.SplitN(line, "=", 2)
+		if len(partsNew) != 2 {
+			log.Log(WARN, "error on config section:", currentSection, "line:", line)
+		}
+
+		key := strings.TrimSpace(partsNew[0])
+		key = strings.TrimSuffix(key, "\"")
+
+		value := strings.TrimSpace(partsNew[1])
+		value = strings.TrimSuffix(value, "\"")
+
+		switch currentSection {
+		case "core":
+			repo.GitConfig.Core[key] = value
+		case "gui":
+			// don't really need gui stuff right now
+		case "pull":
+			// don't store git config pull settings here
+			// git config probably has 'rebase = false'
+		case "remote":
+			test, ok := repo.GitConfig.Remotes[currentName]
+			if !ok {
+				test = new(GitRemote)
+				repo.GitConfig.Remotes[currentName] = test
+			}
+			log.Log(INFO, "switch currentSection", currentSection, currentName)
+			switch key {
+			case "url":
+				if test.Url == value {
+					continue
+				}
+				if test.Url == "" {
+					test.Url = value
+					continue
+				}
+				log.Log(INFO, "error url mismatch", test.Url, value)
+			case "fetch":
+				if test.Fetch == value {
+					continue
+				}
+				if test.Fetch == "" {
+					test.Fetch = value
+					continue
+				}
+				log.Log(INFO, "error fetch mismatch", test.Fetch, value)
+			default:
+				log.Log(INFO, "unknown remote:", line)
+			}
+		case "branch":
+			test, ok := repo.GitConfig.Branches[currentName]
+			if !ok {
+				test = new(GitBranch)
+				repo.GitConfig.Branches[currentName] = test
+				repo.processBranch(currentName)
+			}
+			switch key {
+			case "remote":
+				repo.GitConfig.Branches[currentName].Remote = value
+			case "merge":
+				repo.GitConfig.Branches[currentName].Merge = value
+			default:
+				log.Log(INFO, "error unknown remote:", currentSection, currentName, key, value)
+				log.Log(INFO, "unknown branch:", line)
+			}
+		case "submodule":
+			// test, ok := rs.gitConfig.submodules[currentName]
+			switch key {
+			case "active":
+				// probably 'true' or 'false'
+			case "url":
+				repo.GitConfig.Submodules[currentName] = value
+			default:
+				log.Log(WARN, "unknown submodule line:", line)
+			}
+		case "diff":
+			// test, ok := rs.gitConfig.submodules[currentName]
+			switch key {
+			case "renameLimit":
+				log.Log(INFO, "name:", line)
+			default:
+				log.Log(WARN, "unknown name line:", filename, line)
+			}
+		case "user":
+			// test, ok := rs.gitConfig.submodules[currentName]
+			switch key {
+			case "name":
+				log.Log(INFO, "name:", line)
+			case "email":
+				log.Log(INFO, "email:", line)
+			default:
+				log.Log(WARN, "unknown name line:", filename, line)
+			}
+		case "git-bug":
+			log.Log(WARN, "repo uses git-bug!", filename)
+		default:
+			log.Log(WARN, filename, "unknown line:", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (repo *Repo) processBranch(branch string) {
+	log.Log(INFO, "    ", branch)
+	hash, ok := repo.GitConfig.Hashes[branch]
+	filename := filepath.Join(repo.GetFullPath() + "/.git/refs/heads/" + branch)
+	log.Log(INFO, "      hash: need to open", filename)
+
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Log(WARN, "hash: read failed", filename)
+		return
+	}
+	newhash := strings.TrimSpace(string(data))
+	log.Log(INFO, "      hash:", newhash)
+	repo.GitConfig.Hashes[branch] = newhash
+	if ok {
+		if hash != newhash {
+			log.Log(WARN, "hash changed", hash)
+		}
+	}
+
+	name, _ := repo.gitDescribeByHash(newhash)
+	repo.GitConfig.Versions[newhash] = name
+	log.Log(INFO, "      hash: version", name)
+}

reloadIsTracked.go

@@ -59,7 +59,7 @@ func (repo *Repo) RepoIgnoresGoMod() error {
 		}
 	} else {
 		if ignored {
-			fmt.Printf("%s %s is ignored by Git.\n", repo.GetGoPath(), file)
+			// fmt.Printf("%s %s is ignored by Git.\n", repo.GetGoPath(), file)
 			repo.GoInfo.GitIgnoresGoSum = true
 			return nil
 		}

reloadMtime.go

@@ -119,6 +119,31 @@ func (repo *Repo) changedHead() bool {
 	return true
 }
 
+// check the mtime of the .git/config file
+func (repo *Repo) changedConfig() bool {
+	fname := ".git/config"
+	fileTime := repo.Mtime(fname)
+	if fileTime == nil {
+		// .git/config doesn't exist. something is wrong!
+		log.Info("gitpb .git/config is missing", repo.GetGoPath())
+		return false
+	}
+	mtime := timestamppb.New(*fileTime)
+	pbtime := repo.Times.MtimeConfig
+	if pbtime == nil { // this can happen?
+		repo.Times.MtimeConfig = mtime
+		return true
+	}
+
+	if (pbtime.Seconds == mtime.Seconds) && (pbtime.Nanos == mtime.Nanos) {
+		return false
+	}
+	dur := mtime.AsTime().Sub(pbtime.AsTime())
+	repo.StateChange = fmt.Sprintf("%s changed %s", fname, shell.FormatDuration(dur))
+	repo.Times.MtimeConfig = mtime
+	return true
+}
+
 func (repo *Repo) changedIndex() bool {
 	fname := ".git/index"
 	fileTime := repo.Mtime(fname)
@@ -154,6 +179,9 @@ func (repo *Repo) reloadMtimes() bool {
 	if repo.changedIndex() {
 		changed = true
 	}
+	if repo.changedConfig() {
+		changed = true
+	}
 	if repo.changedDir() {
 		// changed = true
 	}
@@ -171,6 +199,9 @@ func (repo *Repo) DidRepoChange() bool {
 	if repo.didFileChange(".git/index", repo.Times.MtimeIndex) {
 		return true
 	}
+	if repo.didFileChange(".git/config", repo.Times.MtimeConfig) {
+		return true
+	}
 	if repo.didFileChange(".git", repo.Times.MtimeDir) {
 		// todo: do something with CheckDirty()
 		// return true

reloadRepoState.go

@@ -14,16 +14,38 @@ func (repo *Repo) setRepoState() {
 		repo.State = "dirty"
 		return
 	}
-	if repo.GetUserVersion() != repo.GetDevelVersion() {
-		repo.State = "merge to devel"
-		return
+	if repo.GetUserVersion() == "uerr" {
+		// user has not made user branches
+	} else {
+		if repo.GetUserVersion() != repo.GetDevelVersion() {
+			repo.State = "merge to devel"
+			return
+		}
 	}
 	if repo.GetDevelVersion() != repo.GetMasterVersion() {
-		repo.State = "merge to main"
+		if !repo.IsLocalBranch(repo.GetDevelBranchName()) {
+			// the remote devel branch exists but is not checked out
+			repo.State = "no devel branch"
+			return
+		}
+		b1 := repo.CountDiffObjects(repo.GetMasterBranchName(), repo.GetDevelBranchName())
+		if b1 == 0 {
+			repo.State = "merge to main"
+			// log.Info("master vs devel count is normal b1 == 0", b1)
+		} else {
+			repo.State = "DEVEL behind MASTER"
+			// log.Info("master vs devel count b1 != 0", b1)
+			log.Infof("%s devel branch is behind master branch (missing %d commits)\n", repo.GetGoPath(), b1)
+		}
+		return
+	}
+	// if IsGoTagVersionGreater(oldtag string, newtag string) bool {
+	if !IsGoTagVersionGreater(repo.GetLastTag(), repo.GetMasterVersion()) {
+		repo.State = "last tag greater error"
 		return
 	}
 	if repo.GetLastTag() != repo.GetMasterVersion() {
-		repo.State = "unchanged"
+		repo.State = "ready to release"
 		return
 	}
 
@@ -31,8 +53,31 @@ func (repo *Repo) setRepoState() {
 		repo.State = "PERFECT"
 		return
 	}
-	log.Info("Branches are not Perfect", repo.GetFullPath())
-	log.Info("Branches are not Perfect", repo.GetFullPath())
-	log.Info("Branches are not Perfect", repo.GetFullPath())
 	repo.State = "unknown branches"
 }
+
+// returns true if old="v0.2.4" and new="v0.3.3"
+// returns true if equal
+// todo: make all of this smarter someday
+func IsGoTagVersionGreater(oldtag string, newtag string) bool {
+	olda, oldb, oldc := splitInts(oldtag)
+	newa, newb, newc := splitInts(newtag)
+
+	if newa < olda {
+		return false
+	}
+	if newb < oldb {
+		return false
+	}
+	if newc < oldc {
+		return false
+	}
+	return true
+}
+
+// returns true for "v0.2.4" and false for "v0.2.43-asdfj"
+// actually returns false for anything not perfectly versioned
+func IsGoTagPublished(oldtag string, newtag string) bool {
+	// todo
+	return true
+}

reloadRepoType.go

@@ -9,6 +9,8 @@ import (
 	"go.wit.com/log"
 )
 
+// TODO: this needs to be redone in a smarter way
+// to identify which repos have things to build in them
 func (repo *Repo) GetRepoType() string {
 	if repo == nil {
 		return "nil"
@@ -24,9 +26,6 @@ func (repo *Repo) GetRepoType() string {
 	if repo.GoInfo.GoPlugin {
 		return "plugin"
 	}
-	if repo.GoInfo.GoProtobuf {
-		return "protobuf"
-	}
 	if repo.GoInfo.GoBinary {
 		if repo.Exists(".plugin") {
 			log.Warn("gitpb.RepoType() plugin was not set correctly")
@@ -35,6 +34,10 @@ func (repo *Repo) GetRepoType() string {
 		}
 		return "binary"
 	}
+	// binary should always take precidence over libraries that are protobuf's
+	if repo.GoInfo.GoProtobuf {
+		return "protobuf"
+	}
 	if repo.GoInfo.GoLibrary {
 		return "library"
 	}
@@ -48,11 +51,9 @@ func (repo *Repo) setRepoType() {
 	}
 	if repo.Exists(".plugin") {
 		repo.GoInfo.GoPlugin = true
-		return
 	}
 	if ok, _, _ := repo.IsProtobuf(); ok {
 		repo.GoInfo.GoProtobuf = true
-		return
 	}
 	switch repo.goListRepoType() {
 	case "binary":
@@ -70,9 +71,9 @@ func (repo *Repo) goListRepoType() string {
 	// cmd := []string{"go", "list", "-f", "'{{.Name}}'"} // probably use this. this just prints out the package name
 	// cmd := []string{"go", "list", "-f", "'{{.ImportPath}}'"} // returns go.wit.com/lib/protobuf/gitpb
 
-	result := repo.RunQuiet(cmd)
-	if result.Error != nil {
-		log.Warn("go list binary detect failed", result.Error)
+	result, err := repo.RunQuiet(cmd)
+	if err != nil {
+		// log.Info("go list binary detect failed", err)
 		return ""
 	}
 	output := strings.TrimSpace(strings.Join(result.Stdout, "\n"))

reloadTags.go

@@ -11,6 +11,31 @@ import (
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 )
 
+// redo this. use go-git2 ?
+func (repo *Repo) allCommits() error {
+	return nil
+}
+
+/*
+// this is dumb
+func (repo *Repo) AllCommits() error {
+	// tags := []string{"cd", "%(creatordate)", "%(*authordate)", "%(refname)", "%(subject)"}
+	// format := strings.Join(tags, "_,,,_%")
+
+	cmd := []string{"git", "log", "--format=%cd"}
+	result := shell.PathRunQuiet(repo.FullPath, cmd)
+	if result.Error != nil {
+		log.Warn("git for-each-ref error:", result.Error)
+		return result.Error
+	}
+	newest := strings.Join(result.Stdout, "\n")
+	newest = strings.TrimSpace(newest)
+	tmp := getGitDateStamp(newest)
+	repo.Times.NewestCommit = timestamppb.New(tmp)
+	return nil
+}
+*/
+
 // reload the tags
 func (repo *Repo) reloadGitTags() error {
 	// todo: look for changes in the tags?
@@ -67,7 +92,14 @@ func (repo *Repo) reloadGitTags() error {
 		repo.Tags.Append(&newr)
 	}
 
+	// GIT_COMMITTER_DATE="$(git log -1 --format=%cI)" \
+	// GIT_AUTHOR_DATE="$(git log -1 --format=%aI)" \
+	// git am --committer-date-is-author-date < patch.mbox
+
+	// good format for insuring the hashs are identical when using git am
+	// git log -1 --format="%H %aI %cI %an %ae %cn %ce"
 	// also set the repo.NewestCommit
+
 	cmd = []string{"git", "log", "-1", "--format=%cd"}
 	result = shell.PathRunQuiet(repo.FullPath, cmd)
 	if result.Error != nil {
@@ -135,21 +167,28 @@ func (repo *Repo) NewestTag() *GitTag {
 	return nil
 }
 
+// this should just do is.Exists(".git/refs/heads/findname")
 func (repo *Repo) LocalTagExists(findname string) bool {
-	loop := repo.Tags.SortByRefname()
-	for loop.Scan() {
-		ref := loop.Next()
-		// log.Info(repo.GoPath, ref.Refname)
-		if strings.HasPrefix(ref.Refname, "refs/remotes") {
-			continue
-		}
-		_, tagname := filepath.Split(ref.Refname)
-		// log.Info("tag:", path, tagname, "from", repo.GoPath)
-		if tagname == findname {
-			// log.Info("found tag:", path, tagname, "from", repo.GoPath)
-			return true
-		}
+	fname := filepath.Join(".git/refs/heads", findname)
+	if repo.Exists(fname) {
+		return true
 	}
+	/*
+		loop := repo.Tags.SortByRefname()
+		for loop.Scan() {
+			ref := loop.Next()
+			// log.Info(repo.GoPath, ref.Refname)
+			if strings.HasPrefix(ref.Refname, "refs/remotes") {
+				continue
+			}
+			tagname := filepath.Base(ref.Refname)
+			// log.Info("tag:", path, tagname, "from", repo.GoPath)
+			if tagname == findname {
+				// log.Info("found tag:", path, tagname, "from", repo.GoPath)
+				return true
+			}
+		}
+	*/
 	return false
 }
 

repo.common.go

@@ -9,6 +9,23 @@ import (
 	"go.wit.com/log"
 )
 
+// does this repo build a binary?
+func (r *Repo) IsBinary() bool {
+	if r.GoInfo != nil {
+		return r.GoInfo.GetGoBinary()
+	}
+	// todo: detect C & other language binary packages
+	return false
+}
+
+// does this repo build a binary?
+func (r *Repo) IsGoPlugin() bool {
+	if r.GoInfo != nil {
+		return r.GoInfo.GetGoPlugin()
+	}
+	return false
+}
+
 // This returns the list of autogenerated protobuf files
 // it assumes any file *.pb.go is autogenerated
 //

repo.helpers.go

@@ -0,0 +1,19 @@
+// Code generated by go.wit.com/apps/autogenpb DO NOT EDIT.
+// This file was autogenerated with autogenpb v0.0.78 2025-08-31_15:49:04_UTC
+//     go install go.wit.com/apps/autogenpb@latest
+//
+// define which structs (messages) you want to use in the .proto file
+// Then sort.pb.go and marshal.pb.go files are autogenerated
+//
+// autogenpb uses it and has an example .proto file with instructions
+//
+
+package gitpb
+
+func (t *ReposTable) MyMasterBranch() *RepoStringFunc {
+	sf := t.AddStringFunc("master", func(m *Repo) string {
+		return m.MasterBranchName
+	})
+	sf.Width = 30
+	return sf
+}

repo.merge.go

@@ -1,44 +1,93 @@
 package gitpb
 
 import (
+	"fmt"
+
 	"github.com/go-cmd/cmd"
 	"go.wit.com/log"
 )
 
-func (rs *Repo) MergeUserToDevel() (*cmd.Status, error) {
-	startbranch := rs.GetCurrentBranchName()
-	devel := rs.GetDevelBranchName()
-	user := rs.GetUserBranchName()
-
-	log.Info("MergeUserToDevel() checking out", devel, "started on", startbranch, "merge", user)
-
-	var all [][]string
-	all = append(all, []string{"git", "checkout", devel})
-	all = append(all, []string{"git", "merge", user})
-	all = append(all, []string{"git", "push"})
-
-	if result, err := rs.RunStrictAll(all); err != nil {
-		log.Log(WARN, "MergeUserToDevel() failed", rs.GetFullPath())
-		return result, err
+func (r *Repo) MergeToDevel() (*cmd.Status, error) {
+	r.Reload()
+	if r.GetCurrentBranchName() != r.GetDevelBranchName() {
+		return nil, fmt.Errorf("repo not on devel branch")
 	}
+	if r.CheckDirty() {
+		return nil, fmt.Errorf("repo is dirty")
+	}
+	devel := r.GetDevelBranchName()
+	user := r.GetUserBranchName()
+
+	log.Info(r.FullPath, "MergeToDevel() merging from", user, "into", devel)
+
+	cmd := []string{"git", "merge", user}
+	result := r.RunRealtimeVerbose(cmd)
+	if result.Error != nil {
+		log.Log(WARN, "MergeToDevel() failed", r.GetFullPath())
+		return nil, result.Error
+	}
+
+	if !r.IsBranchRemote(devel) {
+		r.Reload() // rescan the repo
+		// devel branch is not remote. do not try 'git push'
+		return nil, nil
+	}
+	if r.GetReadOnly() {
+		r.Reload() // rescan the repo
+		// devel branch is read only. you can not git push
+		return nil, nil
+	}
+
+	// it seems like we have write access. lets find out!
+	cmd = []string{"git", "push"}
+	result = r.RunRealtimeVerbose(cmd)
+	if result.Error != nil {
+		log.Log(WARN, "GitPushToDevel() failed", r.GetFullPath())
+		return nil, result.Error
+	}
+	r.Reload() // rescan the repo
 	return nil, nil
 }
 
-func (rs *Repo) MergeDevelToMaster() (*cmd.Status, error) {
-	startbranch := rs.GetCurrentBranchName()
-	devel := rs.GetDevelBranchName()
-	main := rs.GetMasterBranchName()
+func (r *Repo) MergeToMaster() (*cmd.Status, error) {
+	r.Reload()
 
-	log.Info("MergeDevelToMaster() checking out", main, "started on", startbranch, "merge", devel)
-
-	var all [][]string
-	all = append(all, []string{"git", "checkout", main})
-	all = append(all, []string{"git", "merge", devel})
-	all = append(all, []string{"git", "push"})
-
-	if result, err := rs.RunStrictAll(all); err != nil {
-		log.Log(WARN, "MergeDevelToMaster() failed", rs.GetFullPath())
-		return result, err
+	if r.GetCurrentBranchName() != r.GetMasterBranchName() {
+		return nil, fmt.Errorf("repo not on master branch")
 	}
+	if r.GetReadOnly() {
+		r.Reload() // rescan the repo
+		// master branch is read only. you can not git push
+		return nil, fmt.Errorf("can't merge to master on read only() repos")
+	}
+	if r.CheckDirty() {
+		return nil, fmt.Errorf("repo is dirty")
+	}
+	master := r.GetMasterBranchName()
+	devel := r.GetDevelBranchName()
+
+	log.Info("MergeToMaster() merging from", devel, "into", master)
+
+	cmd := []string{"git", "merge", devel}
+	result := r.RunRealtimeVerbose(cmd)
+	if result.Error != nil {
+		log.Log(WARN, "MergeToMaster() failed", r.GetFullPath())
+		return nil, result.Error
+	}
+
+	if r.GetReadOnly() {
+		r.Reload() // rescan the repo
+		// master branch is read only. you can not git push
+		return nil, nil
+	}
+
+	// it seems like we have write access. lets find out!
+	cmd = []string{"git", "push"}
+	result = r.RunRealtimeVerbose(cmd)
+	if result.Error != nil {
+		log.Log(WARN, "GitPushToMaster() failed", r.GetFullPath())
+		return nil, result.Error
+	}
+	r.Reload() // rescan the repo
 	return nil, nil
 }

repo.new.go

@@ -25,7 +25,8 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 
 	// add a new one here
 	newr := Repo{
-		FullPath: fullpath,
+		FullPath:  fullpath,
+		Namespace: gopath,
 	}
 	newr.Times = new(GitTimes)
 
@@ -34,6 +35,7 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 	// everything happens in here
 	newr.Reload()
 
+	newr.ValidateUTF8()
 	if all.AppendByFullPath(&newr) {
 		// worked
 		return &newr, nil
@@ -49,18 +51,52 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 }
 
 // enforces GoPath is unique
-func (all *Repos) AppendByGoPath(newr *Repo) bool {
-	all.RLock()
+// TODO: deprecate this (?)
+// mutex's should finally work in the autogenpb protobuf code
+/*
+func (all *Repos) AppendByGoPathOld(newr *Repo) bool {
+	// all.RLock()
+	repoMu.RLock()
 
 	for _, r := range all.Repos {
 		if r.GoInfo.GoPath == newr.GoInfo.GoPath {
-			all.RUnlock()
+			// all.RUnlock()
+			repoMu.RUnlock()
 			return false
 		}
 	}
-	all.RUnlock()
+	// all.RUnlock()
+	repoMu.RUnlock()
 
-	// all.Repos = append(all.Repos, newr)
 	all.Append(newr)
 	return true
 }
+*/
+
+func (all *Repos) NewRepo(fullpath string, namespace string) (*Repo, error) {
+	if r := all.FindByFullPath(fullpath); r != nil {
+		log.Info("gitpb.NewRepo() might already have namespace", r.GetNamespace())
+		log.Info("gitpb.NewRepo() already has FullPath", r.FullPath)
+		// already had this gopath
+		return r, errors.New("gitpb.NewRepo() duplicate path " + fullpath)
+	}
+
+	// add a new one here
+	newr := Repo{
+		FullPath:  fullpath,
+		Namespace: namespace,
+	}
+	newr.Times = new(GitTimes)
+
+	// everything happens in here
+	newr.Reload()
+
+	newr.ValidateUTF8()
+	if all.AppendByFullPath(&newr) {
+		// worked
+		return &newr, nil
+	}
+
+	// todo: use Repos.Lock()
+	return nil, errors.New("gitpb.NewRepo() append failed " + fullpath)
+}

repo.proto

@@ -11,63 +11,71 @@ import "google/protobuf/timestamp.proto"; // Import the well-known type for Time
 
 // global settings for autogenpb `autogenpb:mutex`
 
-message Repo {						// `autogenpb:marshal` `autogenpb:2nomutex`
-	string 	fullPath			= 1;	// `autogenpb:unique` `autogenpb:sort` 	// the actual path to the .git directory: '/home/devel/golang.org/x/tools'
-	string 	masterBranchName		= 3; 		// git 'main' or 'master' branch name
-	string 	develBranchName			= 4; 		// whatever the git 'devel' branch name is
-	string 	userBranchName			= 5; 		// whatever your username branch is
-	bool	dirty				= 6;		// if git says things have been changed
-	string	URL				= 7; 		// the URL
-	GitTags tags				= 8;		// known tags
-        GitTimes   times			= 9; 		// store all the mtime values here. these are temporary
-        GoInfo  goInfo  			= 10; 		// put all the go specifcs here
-	GoDeps	goDeps 				= 11;		// what is in the go.sum file
-	string 	currentBranchName		= 12; 		// the branch currently checked out
-	string 	currentBranchVersion		= 13; 		// the branch currently checked out
-	string	lastTag 			= 14; 		// the oldest tag
-	string	targetVersion			= 15; 		// useful during the package release process
-	bool	readOnly			= 16; 		// tracks access to 'git push'
-	string	desc				= 17; 		// what is this repo?
-	string	stateChange           		= 18; 		// used for debugging tool logic
-	string 	masterVersion   		= 19; 		// just store this for now
-	string 	develVersion			= 20; 		//
-	string 	userVersion			= 21; 		//
-	repeated string	dirtyList		= 22;		// store the list from git status --porcelain
-	string	state           		= 23; 		// status or state. useful for building tooling
-	GitTag	currentTag           		= 24; 		// used to examine repo branches
-}
-
-message Repos {				// `autogenpb:marshal` `autogenpb:sort` `autogenpb:2nomutex`
-	string   uuid	        = 1;    // `autogenpb:uuid:8daaeba1-fb1f-4762-ae6e-95a55d352673`
-	string   version        = 2;    // `autogenpb:version:2`
-	repeated Repo repos 	= 3;    // `autogenpb:append`   // generate AppendUnique() function for this
-}
-
 // should it be done this way?
-message GitTimes {		// `autogenpb:2nomutex`
-	google.protobuf.Timestamp lastPull	= 1;		// last time a git pull was done
-	google.protobuf.Timestamp lastUpdate	= 2;		// when was ReloadGit() last done
-	google.protobuf.Timestamp lastDirty	= 3;		// last time CheckDirty() was run
-	google.protobuf.Timestamp mtimeDir	= 4;		// mtime for ./git // maybe useful to track
-	google.protobuf.Timestamp mtimeHead	= 5;		// mtime for ./git/HEAD // these two mtimes allow really fast checks to see if git has changed
-	google.protobuf.Timestamp mtimeIndex    = 6;		// mtime for ./git/HEAD // probably always in sync with HEAD
-	google.protobuf.Timestamp mtimeFetch    = 7;		// mtime for ./git/FETCH_HEAD // last time 'git fetch' or 'git pull' was run on current branch?
-	google.protobuf.Timestamp lastGoDep     = 8;		// mtime for last go.sum scan
-	google.protobuf.Timestamp newestCommit  = 9;		// when the newest commit was
+
+message GitTimes {                                                 // `autogenpb:nomutex`
+        google.protobuf.Timestamp   lastPull                 = 1;  // last time a git pull was done
+        google.protobuf.Timestamp   lastUpdate               = 2;  // when was ReloadGit() last done
+        google.protobuf.Timestamp   lastDirty                = 3;  // last time CheckDirty() was run
+        google.protobuf.Timestamp   mtimeDir                 = 4;  // mtime for ./git // maybe useful to track
+        google.protobuf.Timestamp   mtimeHead                = 5;  // mtime for ./git/HEAD // these two mtimes allow really fast checks to see if git has changed
+        google.protobuf.Timestamp   mtimeIndex               = 6;  // mtime for ./git/HEAD // probably always in sync with HEAD
+        google.protobuf.Timestamp   mtimeFetch               = 7;  // mtime for ./git/FETCH_HEAD // last time 'git fetch' or 'git pull' was run on current branch?
+        google.protobuf.Timestamp   lastGoDep                = 8;  // mtime for last go.sum scan
+        google.protobuf.Timestamp   newestCommit             = 9;  // when the newest commit was
+        google.protobuf.Timestamp   mtimeConfig              = 10; // mtime for the .git/config file
 }
 
 // this is probably better. think about moving to this instead
-message GoInfo {		// `autogenpb:2nomutex`
-	string	goPath				= 1;		// the logical path as used by golang: 'go.wit.com/apps/helloworld'
-	string	desc				= 2; 		// what is this repo?
-	bool	goLibrary 			= 3;		// is this a golang library?
-	bool	goBinary  			= 4;		// is this a golang binary?
-	bool	goPrimitive		 	= 5;		// if this is a golang primitive (only has go.mod)
-	bool	goPlugin			= 6;		// is this a golang plugin?
-	bool	goProtobuf			= 7;		// autogen go files from .proto
-	GoDeps	goDeps 				= 8;		// what is in the go.sum file
-	GoDeps	published			= 9;		// the last published go.mod/go.sum
-	bytes	goMod				= 10; 		// the last go.mod file
-	bytes	goSum				= 11; 		// the last go.sum file
-	bool	gitIgnoresGoSum                 = 12;		// does .gitignore ignore go.mod & go.sum?
+message GoInfo {                                                   // `autogenpb:nomutex`
+        string                      goPath                   = 1;  // the logical path as used by golang: 'go.wit.com/apps/helloworld'
+        string                      desc                     = 2;  // what is this repo?
+        bool                        goLibrary                = 3;  // is this a golang library?
+        bool                        goBinary                 = 4;  // is this a golang binary?
+        bool                        goPrimitive              = 5;  // if this is a golang primitive (only has go.mod)
+        bool                        goPlugin                 = 6;  // is this a golang plugin?
+        bool                        goProtobuf               = 7;  // autogen go files from .proto
+        GoDeps                      goDeps                   = 8;  // what is in the go.sum file
+        GoDeps                      published                = 9;  // the last published go.mod/go.sum
+        bytes                       goMod                    = 10; // the last go.mod file
+        bytes                       goSum                    = 11; // the last go.sum file
+        bool                        gitIgnoresGoSum          = 12; // does .gitignore ignore go.mod & go.sum?
+}
+
+message Repo {                                                     // `autogenpb:marshal` `autogenpb:nomutex`
+        string                      namespace                = 1;  // `autogenpb:unique` `autogenpb:sort` 	// this repo is 'go.wit.com/lib/protobuf/gitpb'
+        string                      fullPath                 = 2;  // `autogenpb:unique` `autogenpb:sort` 	// the OS path to the .git directory: '/home/devel/golang.org/x/tools'
+        string                      masterBranchName         = 3;  // git 'main' or 'master' branch name
+        string                      develBranchName          = 4;  // whatever the git 'devel' branch name is
+        string                      userBranchName           = 5;  // whatever your username branch is
+        bool                        dirty                    = 6;  // if git says things have been changed
+        string                      URL                      = 7;  // the URL
+        GitTags                     tags                     = 8;  // known tags
+        GitTimes                    times                    = 9;  // store all the mtime values here. these are temporary
+        GoInfo                      goInfo                   = 10; // put all the go specifcs here
+        GoDeps                      goDeps                   = 11; // what is in the go.sum file
+        string                      currentBranchName        = 12; // the branch currently checked out
+        string                      currentBranchVersion     = 13; // the branch currently checked out
+        string                      lastTag                  = 14; // the oldest tag
+        string                      targetVersion            = 15; // useful during the package release process
+        bool                        readOnly                 = 16; // tracks access to 'git push'
+        string                      desc                     = 17; // what is this repo?
+        string                      stateChange              = 18; // used for debugging tool logic
+        string                      masterVersion            = 19; // just store this for now
+        string                      develVersion             = 20; //
+        string                      userVersion              = 21; //
+        repeated string             dirtyList                = 22; // store the list from git status --porcelain
+        string                      state                    = 23; // status or state. useful for building tooling
+        GitTag                      currentTag               = 24; // used to examine repo branches
+        GitConfig                   gitConfig                = 25; // protobuf of the current .git/config
+        string                      MasterHash               = 26; // hash of the current master branch
+        string                      DevelHash                = 27; // hash of the current devel branch
+}
+
+message Repos {                                                    // `autogenpb:marshal` `autogenpb:sort` `autogenpb:gui` `autogenpb:nomutex` `autogenpb:http`
+        string                      uuid                     = 1;  // `autogenpb:uuid:8daaeba1-fb1f-4762-ae6e-95a55d352673`
+        string                      version                  = 2;  // `autogenpb:version:v4`
+        repeated Repo               repos                    = 3;  // `autogenpb:append`   // generate AppendUnique() function for this
+        bool                        hasFullScan              = 4;  // a full repo scan has been saved to disk
+        google.protobuf.Timestamp   fullScan                 = 5;  // mtime of the last full scan saved to disk
 }

rill.go

@@ -5,7 +5,6 @@ package gitpb
 
 import (
 	"errors"
-	"strings"
 	sync "sync"
 
 	"github.com/destel/rill"
@@ -17,46 +16,28 @@ var ErrorMissingGitConfig error = errors.New("missing .git/config")
 var ErrorGitPullOnLocal error = errors.New("git pull on local only branch")
 var ErrorGitPullOnDirty error = errors.New("cannot git pull on dirty repo")
 
-func (repo *Repo) GitPull() cmd.Status {
-	currentName := repo.GetCurrentBranchName()
-	if repo.IsOnlyLocalTag(currentName) {
-		var result cmd.Status
-		result.Exit = 21
-		result.Error = ErrorGitPullOnLocal
-		// log.Info("git pull skipped on local only branch", repo.GetGoPath())
-		return result
-	}
+func (repo *Repo) GitPull() (*cmd.Status, error) {
+	/*
+		currentName := repo.GetCurrentBranchName()
+		if repo.IsOnlyLocalTag(currentName) {
+			var result cmd.Status
+			result.Exit = 21
+			result.Error = ErrorGitPullOnLocal
+			// log.Info("git pull skipped on local only branch", repo.GetGoPath())
+			return result
+		}
+	*/
 	var cmd []string
 	cmd = append(cmd, "git", "pull")
-	r := repo.Run(cmd)
-	// output := strings.Join(r.Stdout, "\n")
-	/* notsure why I had this. I think from a long time ago
-	if r.Error != nil {
-		output = "git error_,,,_a_,,,_b_,,,c"
-	}
-	*/
-	/*
-		if r.Error == nil {
-			if r.Exit == 0 {
-				log.Log(WARN, "git pull ran  ", repo.GetGoPath())
-				// log.Log(WARN, "git pull output", output)
-				return r
-			} else {
-				log.Log(WARN, "git pull error", repo.GetGoPath(), strings.Join(r.Stderr, "\n"))
-				return r
-			}
-		}
-		log.Log(WARN, "git pull error", repo.GetGoPath(), r.Error)
-	*/
-	return r
+	return nil, repo.RunVerbose(cmd)
 }
 
 // rill is awesome. long live rill
 // attempt scan with rill
-func (all *Repos) RillGitPull(part1 int, part2 int) map[*Repo]cmd.Status {
+func (all *Repos) RillGitPull(part1 int, part2 int) map[*Repo]*cmd.Status {
 	var lock sync.Mutex
-	var allerr map[*Repo]cmd.Status
-	allerr = make(map[*Repo]cmd.Status)
+	var allerr map[*Repo]*cmd.Status
+	allerr = make(map[*Repo]*cmd.Status)
 
 	// Convert a slice of user IDs into a channel
 	ids := rill.FromSlice(all.Repos, nil)
@@ -72,7 +53,7 @@ func (all *Repos) RillGitPull(part1 int, part2 int) map[*Repo]cmd.Status {
 		counter += 1
 		if repo.CheckDirty() {
 			// log.Info("git pull skipped on dirty repo", repo.GoPath)
-			var result cmd.Status
+			result := new(cmd.Status)
 			result.Error = ErrorGitPullOnDirty
 			lock.Lock()
 			defer lock.Unlock()
@@ -85,10 +66,10 @@ func (all *Repos) RillGitPull(part1 int, part2 int) map[*Repo]cmd.Status {
 				dur = time.Duration((1+rand.Intn(50))*20) * time.Millisecond
 				time.Sleep(dur)
 			*/
-			var result cmd.Status
-			result = repo.GitPull()
+			var result *cmd.Status
+			result, _ = repo.GitPull()
 			log.Info("git pull", repo.GetGoPath())
-			log.Info(strings.Join(result.Stdout, "\n"))
+			// log.Info(strings.Join(result.Stdout, "\n"))
 			lock.Lock()
 			defer lock.Unlock()
 			allerr[repo] = result

shell.go

@@ -25,11 +25,6 @@ func (repo *Repo) Run(cmd []string) cmd.Status {
 	return result
 }
 
-func (repo *Repo) RunQuiet(cmd []string) cmd.Status {
-	result := shell.PathRunQuiet(repo.FullPath, cmd)
-	return result
-}
-
 func (repo *Repo) RunEcho(cmd []string) cmd.Status {
 	result := shell.PathRunQuiet(repo.FullPath, cmd)
 	log.Log(NOW, "cmd:", repo.FullPath, cmd)
@@ -48,41 +43,32 @@ func (repo *Repo) RunRealtime(cmd []string) cmd.Status {
 }
 
 func (repo *Repo) RunRealtimeVerbose(cmd []string) cmd.Status {
-	log.Log(NOW, "Run:", repo.GetFullPath(), cmd)
+	log.Log(NOW, "EXEC: cd", repo.GetFullPath(), ";", cmd)
 	return shell.PathRunRealtime(repo.GetFullPath(), cmd)
 }
 
-// error if result.Error or if result.Exit != 0
-func (repo *Repo) RunStrict(cmd []string) error {
-	return repo.StrictRun(cmd)
-}
-
-func (repo *Repo) StrictRun(cmd []string) error {
-	result := repo.RunQuiet(cmd)
-	if result.Error != nil {
-		log.Warn(repo.GetGoPath(), cmd, "wow. golang is cool. an os.Error:", result.Error)
-		return result.Error
-	}
-	if result.Exit != 0 {
-		log.Warn(cmd, "failed with", result.Exit)
-		return errors.New(fmt.Sprint(cmd, "failed with", result.Exit))
-	}
-	return nil
-}
-
-func (repo *Repo) RunStrictNew(cmd []string) (*cmd.Status, error) {
-	result := repo.RunQuiet(cmd)
+func (repo *Repo) RunQuiet(cmd []string) (*cmd.Status, error) {
+	result := shell.PathRunQuiet(repo.FullPath, cmd)
 	if result.Error != nil {
 		log.Warn(repo.GetGoPath(), cmd, "wow. golang is cool. an os.Error:", result.Error)
 		return &result, result.Error
 	}
 	if result.Exit != 0 {
-		log.Warn(cmd, "failed with", result.Exit)
+		// log.Warn(cmd, "failed with", result.Exit, repo.GetGoPath())
 		return &result, errors.New(fmt.Sprint(cmd, "failed with", result.Exit))
 	}
 	return &result, nil
 }
 
+func (repo *Repo) RunStrict(cmd []string) (*cmd.Status, error) {
+	result, err := repo.RunQuiet(cmd)
+	if err != nil {
+		log.Warn(cmd, "failed with", result.Exit, repo.GetGoPath())
+		return result, errors.New(fmt.Sprint(cmd, "failed with", result.Exit))
+	}
+	return result, nil
+}
+
 func (repo *Repo) Exists(filename string) bool {
 	if repo == nil {
 		return false
@@ -143,30 +129,25 @@ func (repo *Repo) RunAll(all [][]string) bool {
 func (repo *Repo) RunStrictAll(all [][]string) (*cmd.Status, error) {
 	for _, cmd := range all {
 		log.Log(WARN, "doAll() RUNNING: cmd =", cmd)
-		if result, err := repo.RunStrictNew(cmd); err != nil {
+		if result, err := repo.RunStrict(cmd); err != nil {
 			return result, err
 		}
 	}
 	return nil, nil
 }
 
-func (repo *Repo) RunVerbose(cmd []string) (*cmd.Status, error) {
-	log.Info("Running:", repo.GetGoPath(), cmd)
-	r, err := repo.RunStrictNew(cmd)
+func (repo *Repo) RunVerbose(cmd []string) error {
+	// log.Info("EXEC Running:", repo.GetGoPath(), cmd)
+	err := shell.PathExecVerbose(repo.GetFullPath(), cmd)
 	if err != nil {
 		log.Info("Error", cmd, err)
+		return err
 	}
-	for _, line := range r.Stdout {
-		log.Info(line)
-	}
-	for _, line := range r.Stderr {
-		log.Info(line)
-	}
-	return r, err
+	return nil
 }
 
 func (repo *Repo) RunVerboseOnError(cmd []string) (*cmd.Status, error) {
-	r, err := repo.RunStrictNew(cmd)
+	r, err := repo.RunStrict(cmd)
 	if err == nil {
 		return r, err
 	}
@@ -193,3 +174,14 @@ func (repo *Repo) ConstructGitDiffLog(branch1, branch2 string) []string {
 	cmd = append(cmd, branch2)
 	return cmd
 }
+
+// count all objects only in branch1
+func (repo *Repo) CountDiffObjects(branch1, branch2 string) int {
+	cmd := repo.ConstructGitDiffLog(branch1, branch2)
+	r, err := repo.RunVerboseOnError(cmd)
+	if err != nil {
+		return -1
+	}
+	// log.Info("countDiffObjects()", cmd, len(r.Stdout), strings.Join(r.Stdout, " "))
+	return len(r.Stdout)
+}