add debugging printf's

Makefile

@@ -22,7 +22,7 @@ goimports:
 # dump autogenerated files and potential patches
 clean:
 	rm -f *.pb.go go.* *.patch
-	go-mod-clean --purge
+	go-mod-clean purge
 
 #refs.pb.go: refs.proto
 #	cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \

branches.go

@@ -37,9 +37,9 @@ func (repo *Repo) GetRemoteBranches() []string {
 // v0.22.61-15-gbab84d7
 func (repo *Repo) GetHashName(h string) (string, error) {
 	h = strings.TrimSpace(h)
-	log.Info("GetHashName() is looking for", repo.GetGoPath(), h)
+	// log.Info("GetHashName() is looking for", repo.GetGoPath(), h)
 	cmd := []string{"git", "describe", "--tags", h}
-	r, err := repo.RunStrict(cmd)
+	r, err := repo.RunQuiet(cmd)
 	if err != nil {
 		return "", err
 	}

checkout.go

@@ -1,6 +1,7 @@
 package gitpb
 
 import (
+	"os"
 	"path/filepath"
 
 	"go.wit.com/log"
@@ -8,6 +9,20 @@ import (
 
 func (repo *Repo) CheckoutMaster() bool {
 	bName := repo.GetMasterBranchName()
+	if bName == "giterr" {
+		cmd := []string{"git", "checkout", "main"} // todo: figure out main
+		repo.RunVerbose(cmd)
+		os.Exit(-1)
+		// TODO: try to fix this
+		if repo.checkoutBranch("main") {
+			repo.MasterBranchName = "main"
+			return true
+		} else {
+			cmd := []string{"git", "checkout", "main"} // todo: figure out main
+			repo.RunVerbose(cmd)
+			return false
+		}
+	}
 	if repo.checkoutBranch(bName) {
 		return true
 	}

config.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"go.wit.com/lib/protobuf/bugpb"
 	"go.wit.com/log"
 )
 
@@ -26,6 +27,19 @@ func (all *Repos) ConfigSave() error {
 	data, err := all.Marshal()
 	if err != nil {
 		log.Info("gitpb proto.Marshal() failed len", len(data), err)
+		// often this is because strings have invalid UTF-8. This should probably be fixed in the protobuf code
+		if err := all.tryValidate(); err != nil {
+			return err
+		} else {
+			// re-attempt Marshal() here
+			data, err = all.Marshal()
+			if err == nil {
+				// validate & sanitize strings worked
+				log.Info("gitpb.ConfigSave() repos.Marshal() worked len", len(all.Repos), "repos")
+				configWrite("repos.pb", data)
+				return nil
+			}
+		}
 		return err
 	}
 	log.Info("gitpb.ConfigSave() repos.Marshal() worked len", len(all.Repos), "repos")
@@ -33,6 +47,20 @@ func (all *Repos) ConfigSave() error {
 	return nil
 }
 
+func (all *Repos) tryValidate() error {
+
+	err := bugpb.ValidateProtoUTF8(all)
+	if err != nil {
+		log.Printf("Protobuf UTF-8 validation failed: %v\n", err)
+	}
+	if err := bugpb.SanitizeProtoUTF8(all); err != nil {
+		log.Warn("Sanitation failed:", err)
+		// log.Fatalf("Sanitization failed: %v", err)
+		return err
+	}
+	return nil
+}
+
 // load the repos.pb file. I shouldn't really matter if this
 // fails. the file should be autogenerated. This is used
 // locally just for speed

gitTag.byAge.go

@@ -12,7 +12,7 @@ import (
 // todo: probably switch to using slices. new things added in 1.23
 // https://pkg.go.dev/slices
 
-func (all *GitTags) newSort() *GitTagIterator {
+func (all *GitTags) newSort() *GitTagScanner {
 	slices.SortFunc(all.GitTags, func(a, b *GitTag) int {
 		if n := strings.Compare(a.Name, b.Name); n != 0 {
 			return n
@@ -44,12 +44,12 @@ func (all *GitTags) GetAge(name string) time.Time {
 	return newest
 }
 
-func (all *GitTags) SortByAge() *GitTagIterator {
+func (all *GitTags) SortByAge() *GitTagScanner {
 	packs := all.selectAllGitTags()
 
 	sort.Sort(GitTagAge(packs))
 
-	iterator := newGitTagIterator(packs)
+	iterator := newGitTagScanner(packs)
 	return iterator
 }
 

gitTag.common.go

@@ -10,9 +10,7 @@ import (
 func (repo *Repo) DevelHash() string {
 	brname := repo.GetDevelBranchName()
 	refname := "refs/heads/" + brname
-	all := repo.Tags.All()
-	for all.Scan() {
-		tag := all.Next()
+	for tag := range repo.Tags.IterAll() {
 		// log.Info("repo tag", tag.GetHash(), tag.GetRefname())
 		if tag.GetRefname() == refname {
 			return tag.GetHash()
@@ -23,9 +21,7 @@ func (repo *Repo) DevelHash() string {
 
 func (repo *Repo) GetLocalHash(brname string) string {
 	refname := "refs/heads/" + brname
-	all := repo.Tags.All()
-	for all.Scan() {
-		tag := all.Next()
+	for tag := range repo.Tags.IterAll() {
 		// log.Info("repo tag", tag.GetHash(), tag.GetRefname())
 		if tag.GetRefname() == refname {
 			return strings.TrimSpace(tag.GetHash())
@@ -36,9 +32,7 @@ func (repo *Repo) GetLocalHash(brname string) string {
 
 func (repo *Repo) GetRemoteHash(brname string) string {
 	refname := "refs/remotes/origin/" + brname
-	all := repo.Tags.All()
-	for all.Scan() {
-		tag := all.Next()
+	for tag := range repo.Tags.IterAll() {
 		// log.Info("repo tag", tag.GetHash(), tag.GetRefname())
 		if tag.GetRefname() == refname {
 			return strings.TrimSpace(tag.GetHash())
@@ -88,9 +82,7 @@ func (repo *Repo) IsDevelRemote() bool {
 // matter much here yet
 // eventually this will be worked out by forge in some future code that hasn't been made yet
 func (repo *Repo) IsBranch(findname string) bool {
-	loop := repo.Tags.All()
-	for loop.Scan() {
-		t := loop.Next()
+	for t := range repo.Tags.IterAll() {
 		// log.Info("LocalTagExists() tag:", t.Refname)
 
 		tagname := t.Refname
@@ -109,9 +101,7 @@ func (repo *Repo) IsBranch(findname string) bool {
 }
 
 func (repo *Repo) IsLocalBranch(findname string) bool {
-	loop := repo.Tags.All()
-	for loop.Scan() {
-		t := loop.Next()
+	for t := range repo.Tags.IterAll() {
 		if !strings.HasPrefix(t.Refname, "refs/heads") {
 			// log.Info("LocalTagExists() skip tag:", t.Refname)
 			continue
@@ -131,9 +121,7 @@ func (repo *Repo) IsLocalBranch(findname string) bool {
 // finds the newest tag. used for deciding if master needs to be published
 func (repo *Repo) FindLastTag() string {
 	var newest *GitTag
-	all := repo.Tags.All()
-	for all.Scan() {
-		tag := all.Next()
+	for tag := range repo.Tags.IterAll() {
 		if !strings.HasPrefix(tag.GetRefname(), "refs/tags/") {
 			continue
 		}

http.go

@@ -0,0 +1,81 @@
+// Copyright 1994-2025 WIT.COM Inc Licensed GPL 3.0
+
+package gitpb
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/user"
+
+	"go.wit.com/log"
+)
+
+func httpPost(url string, data []byte) ([]byte, error) {
+	var err error
+	var req *http.Request
+
+	req, err = http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
+	log.Info("httpPost() with len", len(data), "url", url)
+	if err != nil {
+		log.Error(err)
+		return nil, err
+	}
+
+	usr, _ := user.Current()
+	req.Header.Set("author", usr.Username)
+	req.Header.Set("hostname", "fixme:hostname")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return []byte("client.Do(req) error"), err
+	}
+	defer resp.Body.Close()
+	// log.Info("httpPost() with len", len(data))
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Error(err)
+		return body, err
+	}
+
+	log.Info("gitpb.httpPost() worked", url)
+	return body, nil
+}
+
+func (r *Repos) SubmitReposPB(url string) (*Repos, error) {
+	msg, err := r.Marshal()
+	if err != nil {
+		log.Info("Marshal() failed:", err)
+		return nil, err
+	}
+	log.Info("proto.Marshal() msg len", len(msg))
+	body, err := httpPost(url, msg)
+	if err != nil {
+		log.Info("httpPost() failed:", err)
+		return nil, err
+	}
+	log.Info("httpPost() worked with url:", url, "len(body) =", len(body))
+	if err := r.Unmarshal(body); err != nil {
+		log.Info("SubmitReposPB() Unmarshal() failed:", err)
+		log.Printf("%s\n", body)
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func (r *Repos) SendPB(w http.ResponseWriter) error {
+	data, err := r.Marshal()
+	if err != nil {
+		log.Info("Marshal() failed:", err)
+		return err
+	}
+	log.Info("SendPB() Marshal() len(data)", len(data))
+	fmt.Fprintf(w, "%s", data)
+
+	return nil
+}

reload.go

@@ -21,6 +21,15 @@ func (repo *Repo) ReloadCheck() error {
 
 // TODO: clean this up more, but it is working now more or less
 func (repo *Repo) Reload() error {
+	// sometimes, on new repos, if .git/HEAD does not exist
+	// defective git daemons or badly configured repos, 'git clone' can fail
+	// if so, 'git fetch origin' can repair the state
+	if !repo.Exists(".git/HEAD") {
+		cmd := []string{"git", "fetch", "origin"}
+		repo.RunVerbose(cmd)
+		cmd = []string{"git", "checkout", "main"} // todo: figure out main
+		repo.RunVerbose(cmd)
+	}
 	// log.Info("in reload", repo.FullPath)
 	repo.Tags = new(GitTags)
 	repo.reloadGitTags()

reloadTags.go

@@ -11,6 +11,31 @@ import (
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 )
 
+// redo this. use go-git2 ?
+func (repo *Repo) allCommits() error {
+	return nil
+}
+
+/*
+// this is dumb
+func (repo *Repo) AllCommits() error {
+	// tags := []string{"cd", "%(creatordate)", "%(*authordate)", "%(refname)", "%(subject)"}
+	// format := strings.Join(tags, "_,,,_%")
+
+	cmd := []string{"git", "log", "--format=%cd"}
+	result := shell.PathRunQuiet(repo.FullPath, cmd)
+	if result.Error != nil {
+		log.Warn("git for-each-ref error:", result.Error)
+		return result.Error
+	}
+	newest := strings.Join(result.Stdout, "\n")
+	newest = strings.TrimSpace(newest)
+	tmp := getGitDateStamp(newest)
+	repo.Times.NewestCommit = timestamppb.New(tmp)
+	return nil
+}
+*/
+
 // reload the tags
 func (repo *Repo) reloadGitTags() error {
 	// todo: look for changes in the tags?
@@ -74,6 +99,7 @@ func (repo *Repo) reloadGitTags() error {
 	// good format for insuring the hashs are identical when using git am
 	// git log -1 --format="%H %aI %cI %an %ae %cn %ce"
 	// also set the repo.NewestCommit
+
 	cmd = []string{"git", "log", "-1", "--format=%cd"}
 	result = shell.PathRunQuiet(repo.FullPath, cmd)
 	if result.Error != nil {

repo.new.go

@@ -25,7 +25,8 @@ func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
 
 	// add a new one here
 	newr := Repo{
-		FullPath: fullpath,
+		FullPath:  fullpath,
+		Namespace: gopath,
 	}
 	newr.Times = new(GitTimes)
 

repo.proto

@@ -12,6 +12,7 @@ import "google/protobuf/timestamp.proto"; // Import the well-known type for Time
 // global settings for autogenpb `autogenpb:mutex`
 
 // should it be done this way?
+
 message GitTimes {                                                 // `autogenpb:nomutex`
         google.protobuf.Timestamp   lastPull                 = 1;  // last time a git pull was done
         google.protobuf.Timestamp   lastUpdate               = 2;  // when was ReloadGit() last done
@@ -42,7 +43,8 @@ message GoInfo {                                                   // `autogenpb
 }
 
 message Repo {                                                     // `autogenpb:marshal` `autogenpb:nomutex`
-        string                      fullPath                 = 1;  // `autogenpb:unique` `autogenpb:sort` 	// the actual path to the .git directory: '/home/devel/golang.org/x/tools'
+        string                      namespace                = 1;  // `autogenpb:unique` `autogenpb:sort` 	// this repo is 'go.wit.com/lib/protobuf/gitpb'
+        string                      fullPath                 = 2;  // `autogenpb:unique` `autogenpb:sort` 	// the OS path to the .git directory: '/home/devel/golang.org/x/tools'
         string                      masterBranchName         = 3;  // git 'main' or 'master' branch name
         string                      develBranchName          = 4;  // whatever the git 'devel' branch name is
         string                      userBranchName           = 5;  // whatever your username branch is
@@ -70,7 +72,7 @@ message Repo {                                                     // `autogenpb
 
 message Repos {                                                    // `autogenpb:marshal` `autogenpb:sort` `autogenpb:nomutex` `autogenpb:gui`
         string                      uuid                     = 1;  // `autogenpb:uuid:8daaeba1-fb1f-4762-ae6e-95a55d352673`
-        string                      version                  = 2;  // `autogenpb:version:v3`
+        string                      version                  = 2;  // `autogenpb:version:v4`
         repeated Repo               repos                    = 3;  // `autogenpb:append`   // generate AppendUnique() function for this
         bool                        hasFullScan              = 4;  // a full repo scan has been saved to disk
         google.protobuf.Timestamp   fullScan                 = 5;  // mtime of the last full scan saved to disk