missing \n in Printf()

Makefile

@@ -22,7 +22,7 @@ goimports:
 # dump autogenerated files and potential patches
 clean:
 	rm -f *.pb.go go.* *.patch
-	go-mod-clean --purge
+	go-mod-clean purge
 
 #refs.pb.go: refs.proto
 #	cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \

branches.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"go.wit.com/log"
+	"github.com/go-cmd/cmd"
 )
 
 // returns true if 'git pull' will work
@@ -72,13 +72,40 @@ func (repo *Repo) DeleteLocalDevelBranch() error {
 	if !repo.IsDevelRemote() {
 		return fmt.Errorf("no remote branch")
 	}
+
 	b1 := repo.CountDiffObjects(branch, remote) // should be zero
 	if b1 == 0 {
 		cmd := []string{"git", "branch", "-D", repo.GetDevelBranchName()}
-		log.Info("DEVEL IS IN REMOTE", repo.GetGoPath(), cmd)
-		err := repo.RunVerbose(cmd)
+		_, err := repo.RunVerboseOnError(cmd)
 		return err
 	} else {
 		return fmt.Errorf("local branch has patches not in remote")
 	}
 }
+
+// makes a local branch based off of the master branch
+// (unless a remote devel branch exists. then it uses that)
+func (repo *Repo) MakeLocalDevelBranch() (*cmd.Status, error) {
+	branch := repo.GetDevelBranchName()
+	if branch == "" {
+		// hard coded default
+		branch = "devel"
+	}
+
+	if repo.Exists(filepath.Join(".git/refs/heads", branch)) {
+		// local devel branch already exists
+		return nil, nil
+	}
+
+	if repo.Exists(filepath.Join(".git/refs/remotes/origin", branch)) {
+		// remote devel branch exists, but local does not
+		cmd := []string{"git", "checkout", branch}
+		return repo.RunVerboseOnError(cmd)
+	}
+
+	master := repo.GetMasterBranchName()
+	cmd := []string{"git", "branch", branch, master}
+	repo.RunVerboseOnError(cmd)
+	cmd = []string{"git", "checkout", branch}
+	return repo.RunVerboseOnError(cmd)
+}

checkout.go

@@ -1,6 +1,9 @@
 package gitpb
 
 import (
+	"fmt"
+	"os"
+	"os/user"
 	"path/filepath"
 
 	"go.wit.com/log"
@@ -8,6 +11,20 @@ import (
 
 func (repo *Repo) CheckoutMaster() bool {
 	bName := repo.GetMasterBranchName()
+	if bName == "giterr" {
+		cmd := []string{"git", "checkout", "main"} // todo: figure out main
+		repo.RunVerboseOnError(cmd)
+		os.Exit(-1)
+		// TODO: try to fix this
+		if repo.checkoutBranch("main") {
+			repo.MasterBranchName = "main"
+			return true
+		} else {
+			cmd := []string{"git", "checkout", "main"} // todo: figure out main
+			repo.RunVerboseOnError(cmd)
+			return false
+		}
+	}
 	if repo.checkoutBranch(bName) {
 		return true
 	}
@@ -26,12 +43,20 @@ func (repo *Repo) CheckoutDevel() bool {
 
 func (repo *Repo) CheckoutUser() error {
 	bName := repo.GetUserBranchName()
-	// log.Info("attempting checkout user", repo.GetGoPath(), bName)
-	err := repo.createUserBranchNew(bName)
-	if err != nil {
-		log.Info("attempting checkout user error", repo.GetGoPath(), bName, err)
+	if bName == "uerr" {
+		usr, _ := user.Current()
+		repo.SetUserBranchName(usr.Username)
+		bName = usr.Username
+		log.Info("gitpb CheckoutUser() somehow got user 'uerr'")
 	}
-	return err
+
+	return repo.createUserBranch(bName)
+	/*
+		if err != nil {
+			log.Info("attempting checkout user error", repo.GetGoPath(), bName, err)
+		}
+		return err
+	*/
 }
 
 func (repo *Repo) BranchExists(bName string) bool {
@@ -67,16 +92,16 @@ func (repo *Repo) checkoutBranch(bName string) bool {
 	return true
 }
 
-func (repo *Repo) createUserBranchNew(branch string) error {
-	if branch == "" || branch == "uerr" {
-		log.Info("forge.gitpb logic err. branch name was:", branch)
-		return nil
+// actually creates a local user branch
+func (repo *Repo) createUserBranch(branch string) error {
+	if branch == "" {
+		// get the username here?
+		return fmt.Errorf("gitpb createuserBranch() logic err. git branch name can not be blank")
 	}
 	if repo.IsDirty() {
 		// never change repos on dirty branches
-		return nil
+		return fmt.Errorf("repo is dirty")
 	}
-	// log.Info("forge.gitpb look for branch name was:", branch, repo.GetGoPath())
 
 	if repo.Exists(filepath.Join(".git/refs/heads", branch)) {
 		var err error
@@ -110,8 +135,6 @@ func (repo *Repo) createUserBranchNew(branch string) error {
 		// return fmt.Errorf("repo must be on devel branch %s", repo.GetGoPath())
 	}
 
-	// log.Info("forge.gitpb try to create", branch, repo.GetGoPath())
-
 	// create the branch from devel
 	cmd := []string{"git", "branch", branch}
 	if _, err := repo.RunVerboseOnError(cmd); err != nil {

config.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"go.wit.com/lib/protobuf/bugpb"
 	"go.wit.com/log"
 )
 
@@ -26,6 +27,19 @@ func (all *Repos) ConfigSave() error {
 	data, err := all.Marshal()
 	if err != nil {
 		log.Info("gitpb proto.Marshal() failed len", len(data), err)
+		// often this is because strings have invalid UTF-8. This should probably be fixed in the protobuf code
+		if err := all.tryValidate(); err != nil {
+			return err
+		} else {
+			// re-attempt Marshal() here
+			data, err = all.Marshal()
+			if err == nil {
+				// validate & sanitize strings worked
+				log.Info("gitpb.ConfigSave() repos.Marshal() worked len", len(all.Repos), "repos")
+				configWrite("repos.pb", data)
+				return nil
+			}
+		}
 		return err
 	}
 	log.Info("gitpb.ConfigSave() repos.Marshal() worked len", len(all.Repos), "repos")
@@ -33,6 +47,39 @@ func (all *Repos) ConfigSave() error {
 	return nil
 }
 
+// todo: move this to Marshal() functions automatically in autogenpb?
+func (repo *Repo) ValidateUTF8() error {
+	if _, err := repo.Marshal(); err == nil {
+		// exit if Marshal() works
+		return nil
+	} else {
+		log.Printf("%s repo.Marshal() failed: %v\n", repo.GetFullPath(), err)
+	}
+	// you only need to do this if Marshal() fails
+	err := bugpb.ValidateProtoUTF8(repo)
+	if err != nil {
+		log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
+	}
+	if err := bugpb.SanitizeProtoUTF8(repo); err != nil {
+		log.Warn("Sanitation failed:", err)
+		return err
+	}
+	return nil
+}
+
+func (all *Repos) tryValidate() error {
+	err := bugpb.ValidateProtoUTF8(all)
+	if err != nil {
+		log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
+	}
+	if err := bugpb.SanitizeProtoUTF8(all); err != nil {
+		log.Warn("Sanitation failed:", err)
+		// log.Fatalf("Sanitization failed: %v", err)
+		return err
+	}
+	return nil
+}
+
 // load the repos.pb file. I shouldn't really matter if this
 // fails. the file should be autogenerated. This is used
 // locally just for speed
@@ -72,9 +119,12 @@ func (all *Repos) ConfigLoad() error {
 
 func deleteProtobufFile(filename string) {
 	log.Log(WARN, "The protobuf file format has changed for", filename)
-	log.Log(WARN, "You must delete", filename)
-	log.Log(WARN, "This file will be recreated")
-	os.Exit(-1)
+	log.Log(WARN, "Deleting old file:", filename)
+	log.Log(WARN, "This file will be recreated on the next run.")
+	err := os.Remove(filename)
+	if err != nil {
+		log.Log(WARN, "failed to remove old protobuf file", "err", err)
+	}
 }
 
 func (all *Repos) sampleConfig() {

currentVersions.go

@@ -145,7 +145,7 @@ func (repo *Repo) gitVersionByName(name string) (string, error) {
 	}
 	if !repo.IsBranch(name) {
 		// tag does not exist
-		log.Log(WARN, "LocalTagExists()", name, "did not exist")
+		// log.Log(WARN, "LocalTagExists()", name, "did not exist")
 		return "", errors.New("gitDescribeByName() git fatal: Not a valid object name: " + name)
 	}
 	cmd := []string{"git", "describe", "--tags", "--always", name}

gitTag.byAge.go

@@ -12,7 +12,7 @@ import (
 // todo: probably switch to using slices. new things added in 1.23
 // https://pkg.go.dev/slices
 
-func (all *GitTags) newSort() *GitTagIterator {
+func (all *GitTags) newSort() *GitTagScanner {
 	slices.SortFunc(all.GitTags, func(a, b *GitTag) int {
 		if n := strings.Compare(a.Name, b.Name); n != 0 {
 			return n
@@ -44,12 +44,12 @@ func (all *GitTags) GetAge(name string) time.Time {
 	return newest
 }
 
-func (all *GitTags) SortByAge() *GitTagIterator {
+func (all *GitTags) SortByAge() *GitTagScanner {
 	packs := all.selectAllGitTags()
 
 	sort.Sort(GitTagAge(packs))
 
-	iterator := newGitTagIterator(packs)
+	iterator := newGitTagScanner(packs)
 	return iterator
 }
 

goDep.parseGoSum.go

@@ -14,8 +14,8 @@ import (
 	"go.wit.com/log"
 )
 
-// reads and parses the go.sum file
-// does not change anything
+// reads and parses the go.sum file into a protobuf struct
+// this function isn't supposed to change anything, just parse the existing files
 func (repo *Repo) ParseGoSum() bool {
 	// empty out what was there before
 	repo.GoDeps = nil
@@ -75,7 +75,7 @@ func (repo *Repo) ParseGoSum() bool {
 
 // attempt to parse go.* files in a directory
 func GoSumParseDir(moddir string) (*GoDeps, error) {
-	isprim, err := computePrimitiveNew(moddir)
+	isprim, err := computePrimitive(moddir)
 	if err != nil {
 		// "go mod init" failed
 		return nil, err
@@ -85,14 +85,14 @@ func GoSumParseDir(moddir string) (*GoDeps, error) {
 		return nil, nil
 	}
 	// go.sum exists. parse the go.sum file
-	return parseGoSumNew(moddir)
+	return parseGoSum(moddir)
 }
 
 // Detect a 'Primitive' package. Sets the isPrimitive flag
 // will return true if the repo is truly not dependent on _anything_ else
 // like spew or lib/widget
 // it assumes 'go mod init' and 'go mod tidy' ran without error
-func computePrimitiveNew(moddir string) (bool, error) {
+func computePrimitive(moddir string) (bool, error) {
 	// go mod init & go mod tidy ran without errors
 	log.Log(INFO, "isPrimitiveGoMod()", moddir)
 	gomod, err := os.Open(filepath.Join(moddir, "go.mod"))
@@ -131,7 +131,8 @@ func computePrimitiveNew(moddir string) (bool, error) {
 	return true, nil
 }
 
-func parseGoSumNew(moddir string) (*GoDeps, error) {
+// parse the go.sum file into a protobuf
+func parseGoSum(moddir string) (*GoDeps, error) {
 	godeps := new(GoDeps)
 
 	tmp, err := os.Open(filepath.Join(moddir, "go.sum"))

http.go

@@ -0,0 +1,81 @@
+// Copyright 1994-2025 WIT.COM Inc Licensed GPL 3.0
+
+package gitpb
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/user"
+
+	"go.wit.com/log"
+)
+
+func httpPost(url string, data []byte) ([]byte, error) {
+	var err error
+	var req *http.Request
+
+	req, err = http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
+	log.Info("httpPost() with len", len(data), "url", url)
+	if err != nil {
+		log.Error(err)
+		return nil, err
+	}
+
+	usr, _ := user.Current()
+	req.Header.Set("author", usr.Username)
+	req.Header.Set("hostname", "fixme:hostname")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return []byte("client.Do(req) error"), err
+	}
+	defer resp.Body.Close()
+	// log.Info("httpPost() with len", len(data))
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Error(err)
+		return body, err
+	}
+
+	log.Info("gitpb.httpPost() worked", url)
+	return body, nil
+}
+
+func (r *Repos) SubmitReposPB(url string) (*Repos, error) {
+	msg, err := r.Marshal()
+	if err != nil {
+		log.Info("Marshal() failed:", err)
+		return nil, err
+	}
+	log.Info("proto.Marshal() msg len", len(msg))
+	body, err := httpPost(url, msg)
+	if err != nil {
+		log.Info("httpPost() failed:", err)
+		return nil, err
+	}
+	log.Info("httpPost() worked with url:", url, "len(body) =", len(body))
+	if err := r.Unmarshal(body); err != nil {
+		log.Info("SubmitReposPB() Unmarshal() failed:", err)
+		log.Printf("%s\n", body)
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func (r *Repos) SendPB(w http.ResponseWriter) error {
+	data, err := r.Marshal()
+	if err != nil {
+		log.Info("Marshal() failed:", err)
+		return err
+	}
+	log.Info("SendPB() Marshal() len(data)", len(data))
+	fmt.Fprintf(w, "%s", data)
+
+	return nil
+}

reload.go

@@ -21,6 +21,15 @@ func (repo *Repo) ReloadCheck() error {
 
 // TODO: clean this up more, but it is working now more or less
 func (repo *Repo) Reload() error {
+	// sometimes, on new repos, if .git/HEAD does not exist
+	// defective git daemons or badly configured repos, 'git clone' can fail
+	// if so, 'git fetch origin' can repair the state
+	if !repo.Exists(".git/HEAD") {
+		cmd := []string{"git", "fetch", "origin"}
+		repo.RunVerbose(cmd)
+		cmd = []string{"git", "checkout", "main"} // todo: figure out main
+		repo.RunVerbose(cmd)
+	}
 	// log.Info("in reload", repo.FullPath)
 	repo.Tags = new(GitTags)
 	repo.reloadGitTags()
@@ -47,6 +56,7 @@ func (repo *Repo) Reload() error {
 
 	// LastUpdate should always be the newest time
 	repo.Times.LastUpdate = timestamppb.New(time.Now())
+	repo.ValidateUTF8()
 	return nil
 }
 

reloadBranches.go

@@ -63,7 +63,7 @@ func (repo *Repo) CheckBranches() bool {
 		if hash == hashCheck {
 			// log.Info("notsure why this git show is here", hash)
 		} else {
-			log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
+			// log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
 			perfect = false
 		}
 	}
@@ -170,7 +170,7 @@ func (repo *Repo) ExamineBranches() *GitTag {
 		if hash == hashCheck {
 			// log.Info("notsure why this git show is here", hash)
 		} else {
-			log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
+			// log.Printf("UNKNOWN BRANCH %-50s %s %s %s\n", repo.GetFullPath(), r.Stdout, cmd, b)
 			tag := new(GitTag)
 			tag.Refname = b
 			tag.Hash = hash

reloadGitConfig.go

@@ -138,8 +138,18 @@ func (repo *Repo) readGitConfig() error {
 			default:
 				log.Log(WARN, "unknown submodule line:", line)
 			}
+		case "user":
+			// test, ok := rs.gitConfig.submodules[currentName]
+			switch key {
+			case "name":
+				log.Log(INFO, "name:", line)
+			case "email":
+				log.Log(INFO, "email:", line)
+			default:
+				log.Log(WARN, "unknown name line:", filename, line)
+			}
 		default:
-			log.Log(WARN, "unknown line:", line)
+			log.Log(WARN, "unknown line:", filename, line)
 		}
 	}
 

reloadIsTracked.go

@@ -59,7 +59,7 @@ func (repo *Repo) RepoIgnoresGoMod() error {
 		}
 	} else {
 		if ignored {
-			fmt.Printf("%s %s is ignored by Git.\n", repo.GetGoPath(), file)
+			// fmt.Printf("%s %s is ignored by Git.\n", repo.GetGoPath(), file)
 			repo.GoInfo.GitIgnoresGoSum = true
 			return nil
 		}

reloadRepoState.go

@@ -33,9 +33,9 @@ func (repo *Repo) setRepoState() {
 			repo.State = "merge to main"
 			// log.Info("master vs devel count is normal b1 == 0", b1)
 		} else {
-			repo.State = "DEVEL < MASTER"
+			repo.State = "DEVEL behind MASTER"
 			// log.Info("master vs devel count b1 != 0", b1)
-			log.Info("SERIOUS ERROR. DEVEL BRANCH NEEDS MERGE FROM MASTER b1 ==", b1, repo.GetGoPath())
+			log.Infof("%s devel branch is behind master branch (missing %d commits)\n", repo.GetGoPath(), b1)
 		}
 		return
 	}
@@ -53,9 +53,6 @@ func (repo *Repo) setRepoState() {
 		repo.State = "PERFECT"
 		return
 	}
-	log.Info("Branches are not Perfect", repo.GetFullPath())
-	log.Info("Branches are not Perfect", repo.GetFullPath())
-	log.Info("Branches are not Perfect", repo.GetFullPath())
 	repo.State = "unknown branches"
 }
 

reloadTags.go

@@ -11,6 +11,13 @@ import (
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 )
 
+// redo this. use go-git2 ?
+func (repo *Repo) allCommits() error {
+	return nil
+}
+
+/*
+// this is dumb
 func (repo *Repo) AllCommits() error {
 	// tags := []string{"cd", "%(creatordate)", "%(*authordate)", "%(refname)", "%(subject)"}
 	// format := strings.Join(tags, "_,,,_%")
@@ -27,6 +34,7 @@ func (repo *Repo) AllCommits() error {
 	repo.Times.NewestCommit = timestamppb.New(tmp)
 	return nil
 }
+*/
 
 // reload the tags
 func (repo *Repo) reloadGitTags() error {

repo.new.go

@@ -25,7 +25,8 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 
 	// add a new one here
 	newr := Repo{
-		FullPath: fullpath,
+		FullPath:  fullpath,
+		Namespace: gopath,
 	}
 	newr.Times = new(GitTimes)
 
@@ -34,6 +35,7 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 	// everything happens in here
 	newr.Reload()
 
+	newr.ValidateUTF8()
 	if all.AppendByFullPath(&newr) {
 		// worked
 		return &newr, nil
@@ -49,7 +51,10 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 }
 
 // enforces GoPath is unique
-func (all *Repos) AppendByGoPath(newr *Repo) bool {
+// TODO: deprecate this (?)
+// mutex's should finally work in the autogenpb protobuf code
+/*
+func (all *Repos) AppendByGoPathOld(newr *Repo) bool {
 	// all.RLock()
 	repoMu.RLock()
 
@@ -63,7 +68,35 @@ func (all *Repos) AppendByGoPath(newr *Repo) bool {
 	// all.RUnlock()
 	repoMu.RUnlock()
 
-	// all.Repos = append(all.Repos, newr)
 	all.Append(newr)
 	return true
 }
+*/
+
+func (all *Repos) NewRepo(fullpath string, namespace string) (*Repo, error) {
+	if r := all.FindByFullPath(fullpath); r != nil {
+		log.Info("gitpb.NewRepo() might already have namespace", r.GetNamespace())
+		log.Info("gitpb.NewRepo() already has FullPath", r.FullPath)
+		// already had this gopath
+		return r, errors.New("gitpb.NewRepo() duplicate path " + fullpath)
+	}
+
+	// add a new one here
+	newr := Repo{
+		FullPath:  fullpath,
+		Namespace: namespace,
+	}
+	newr.Times = new(GitTimes)
+
+	// everything happens in here
+	newr.Reload()
+
+	newr.ValidateUTF8()
+	if all.AppendByFullPath(&newr) {
+		// worked
+		return &newr, nil
+	}
+
+	// todo: use Repos.Lock()
+	return nil, errors.New("gitpb.NewRepo() append failed " + fullpath)
+}

repo.proto

@@ -12,6 +12,7 @@ import "google/protobuf/timestamp.proto"; // Import the well-known type for Time
 // global settings for autogenpb `autogenpb:mutex`
 
 // should it be done this way?
+
 message GitTimes {                                                 // `autogenpb:nomutex`
         google.protobuf.Timestamp   lastPull                 = 1;  // last time a git pull was done
         google.protobuf.Timestamp   lastUpdate               = 2;  // when was ReloadGit() last done
@@ -42,7 +43,8 @@ message GoInfo {                                                   // `autogenpb
 }
 
 message Repo {                                                     // `autogenpb:marshal` `autogenpb:nomutex`
-        string                      fullPath                 = 1;  // `autogenpb:unique` `autogenpb:sort` 	// the actual path to the .git directory: '/home/devel/golang.org/x/tools'
+        string                      namespace                = 1;  // `autogenpb:unique` `autogenpb:sort` 	// this repo is 'go.wit.com/lib/protobuf/gitpb'
+        string                      fullPath                 = 2;  // `autogenpb:unique` `autogenpb:sort` 	// the OS path to the .git directory: '/home/devel/golang.org/x/tools'
         string                      masterBranchName         = 3;  // git 'main' or 'master' branch name
         string                      develBranchName          = 4;  // whatever the git 'devel' branch name is
         string                      userBranchName           = 5;  // whatever your username branch is
@@ -68,9 +70,9 @@ message Repo {                                                     // `autogenpb
         GitConfig                   gitConfig                = 25; // protobuf of the current .git/config
 }
 
-message Repos {                                                    // `autogenpb:marshal` `autogenpb:sort` `autogenpb:nomutex` `autogenpb:gui`
+message Repos {                                                    // `autogenpb:marshal` `autogenpb:sort` `autogenpb:gui` `autogenpb:nomutex`
         string                      uuid                     = 1;  // `autogenpb:uuid:8daaeba1-fb1f-4762-ae6e-95a55d352673`
-        string                      version                  = 2;  // `autogenpb:version:v3`
+        string                      version                  = 2;  // `autogenpb:version:v4`
         repeated Repo               repos                    = 3;  // `autogenpb:append`   // generate AppendUnique() function for this
         bool                        hasFullScan              = 4;  // a full repo scan has been saved to disk
         google.protobuf.Timestamp   fullScan                 = 5;  // mtime of the last full scan saved to disk

shell.go

@@ -137,7 +137,7 @@ func (repo *Repo) RunStrictAll(all [][]string) (*cmd.Status, error) {
 }
 
 func (repo *Repo) RunVerbose(cmd []string) error {
-	log.Info("EXEC Running:", repo.GetGoPath(), cmd)
+	// log.Info("EXEC Running:", repo.GetGoPath(), cmd)
 	err := shell.PathExecVerbose(repo.GetFullPath(), cmd)
 	if err != nil {
 		log.Info("Error", cmd, err)