finally a rewrite of the old junky scanner

doDirty.go

@@ -23,8 +23,7 @@ func (f *Forge) CheckDirtyQuiet() {
 	for _, s := range stats {
 		if s.Err == nil {
 		} else {
-			// log.Info("forge SetConfigSave(true)")
+			config.SetChanged("repos", true)
-			f.SetConfigSave(true)
 			changed = true
 		}
 	}
@@ -46,7 +45,7 @@ func (f *Forge) CheckDirty() *gitpb.Repos {
 		if s.Err == nil {
 		} else {
 			log.Info(i, s.Err)
-			f.SetConfigSave(true)
+			config.SetChanged("repos", true)
 		}
 	}
 
@@ -74,16 +73,12 @@ func doCheckDirty(repo *gitpb.Repo) error {
 			// nothing changed
 		} else {
 			log.Info("Repo changed to clean", repo.FullPath)
-			config.SetChanged("repos", true)
 			return log.Errorf("%s repo changed to clean", repo.FullPath)
-			// f.SetConfigSave(true)
 		}
 	} else {
 		if repo.CheckDirty() {
 			log.Info("Repo changed to dirty", repo.FullPath)
-			config.SetChanged("repos", true)
 			return log.Errorf("%s repo changed to dirty", repo.FullPath)
-			// f.SetConfigSave(true)
 		} else {
 			// nothing changed
 		}

forgeConfig.proto

@@ -62,6 +62,8 @@ message ForgeConfigs {                                         // `autogenpb:mar
         string                      PatchDir             = 13; // patch dir
         string                      ForgeURL             = 14; // forge URL
         string                      Filename             = 15; // filename of the config file
+        int32                       rillX                = 16; // used by rill
+        int32                       rillY                = 17; // used by rill
 }
 
 // this generic message is used by autogen to identify and

goSrcFind.go

@@ -4,12 +4,6 @@ package forgepb
 // If there is a go.work file in your parent, that directory will be returned
 // otherwise, return ~/go/src
 
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
 func (f *Forge) GetHome() string {
 	return f.Config.ReposDir
 }
@@ -70,6 +64,7 @@ func (f *Forge) goWorkExists() bool {
 }
 */
 
+/*
 func digup(path string) (string, error) {
 	for {
 		workFilePath := filepath.Join(path, "go.work")
@@ -88,3 +83,4 @@ func digup(path string) (string, error) {
 
 	return "", fmt.Errorf("no go.work file found")
 }
+*/

goSrcScan.go

@@ -1,50 +1,14 @@
 package forgepb
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
-	"github.com/destel/rill"
-	"go.wit.com/lib/protobuf/gitpb"
 	"go.wit.com/log"
 )
 
-func reloadCheck(repo *gitpb.Repo) error {
+/*
-	if err := repo.ReloadCheck(); err != nil {
-		log.Infof("%s reload() says %v\n", repo.FullPath, err)
-		return err
-	}
-	return nil
-}
-
-func (f *Forge) TestScan() error {
-	f.Repos = gitpb.NewRepos()
-	dirs, err := gitDirectoriesNew(f.Config.ReposDir)
-	if err != nil {
-		return err
-	}
-	for i, fullpath := range dirs {
-		repo, err := gitpb.NewRepo(fullpath)
-		if err != nil {
-			log.Info("ReAdd() error", fullpath, err)
-		}
-		log.Info(i, "worked", repo.FullPath)
-		repo = f.Repos.Append(repo)
-		f.VerifyBranchNames(repo)
-		if f.Config.IsReadOnly(repo.GetGoPath()) {
-			repo.ReadOnly = true
-		}
-		repo.ReloadCheck()
-		if i > 5 {
-			break
-		}
-	}
-	return nil
-}
-
 func (f *Forge) ScanGoSrc() (bool, error) {
 	dirs, err := gitDirectoriesNew(f.Config.ReposDir)
 	if err != nil {
@@ -56,18 +20,11 @@ func (f *Forge) ScanGoSrc() (bool, error) {
 		if stat.Err == nil {
 			continue
 		}
-		f.SetConfigSave(true)
+		config.SetChanged("repos", true)
-		/*
-			dur := stat.End.Sub(stat.Start)
-			if dur > time.Second {
-				log.Infof("%s checkRemoteBranches() took a long time (%s) (err=%v)\n", path, shell.FormatDuration(dur), stat.Err)
-			}
-		*/
 	}
 
 	var gopaths []string
 	for _, dir := range dirs {
-		// log.Info("forge.ScanGoSrc()", dir)
 		if strings.HasPrefix(dir, f.Config.ReposDir) {
 			gopath := strings.TrimPrefix(dir, f.Config.ReposDir)
 			gopath = strings.Trim(gopath, "/")
@@ -88,7 +45,7 @@ func (f *Forge) ScanGoSrc() (bool, error) {
 	}
 	if newcount != 0 {
 		log.Info("forge go src scan found", newcount, "repos")
-		f.SetConfigSave(true)
+		config.SetChanged("repos", true)
 	}
 	return true, err
 }
@@ -103,63 +60,6 @@ func (f *Forge) ScanDir(dir string) *gitpb.Repo {
 	return repo
 }
 
-// doesn't enter the directory any further when it finds a .git/
-// not stupid like my old version
-func gitDirectoriesNew(srcDir string) ([]string, error) {
-	var all []string
-	var trip bool
-	err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, err error) error {
-		if err != nil {
-			// Handle possible errors, like permission issues
-			fmt.Fprintf(os.Stderr, "error accessing path %q: %v\n", path, err)
-			return err
-		}
-
-		if d.IsDir() {
-			// log.Info("path is dir", path)
-		} else {
-			_, fname := filepath.Split(path)
-			switch fname {
-			case "repos.pb":
-			case "go.work":
-			case "go.work.last":
-			case "go.work.sum":
-			default:
-				// todo: figure out a way to do padding for init()
-				if trip == false {
-					log.Info("WARNING:")
-				}
-				log.Info("WARNING: you have an untracked file outside of any .git repository:", path)
-				trip = true
-			}
-			return nil
-		}
-
-		gitdir := filepath.Join(path, ".git")
-		_, err2 := os.Stat(gitdir)
-		if !os.IsNotExist(err2) {
-			all = append(all, path)
-			return filepath.SkipDir
-		}
-		return nil
-	})
-	//
-	// probably always leave this here forever
-	// this check, along with CheckDirty() makes sure you can safely delete ~/go/src or the go.work directory
-	// because everything is either checked in or deleted. An important thing to know!
-	if trip {
-		log.Info("WARNING:")
-		log.Info("WARNING: there isn't a way to disable this warning yet")
-		log.Info("WARNING: probably this is a good thing however. you don't want to leave files outside of git repos here")
-		log.Info("WARNING: so this warning should probably stay")
-		log.Info("WARNING:")
-		log.Info("WARNING: this also might mean you put these files here because you are actively working on them")
-		log.Info("WARNING: and you don't want to forget about them")
-		log.Info("WARNING:")
-	}
-	return all, err
-}
-
 func gitDirectoriesOld(srcDir string) ([]string, error) {
 	var all []string
 	err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
@@ -246,3 +146,61 @@ func (f *Forge) ReAdd(repo *gitpb.Repo) (*gitpb.Repo, error) {
 	}
 	return repo, err
 }
+*/
+
+// doesn't enter the directory any further when it finds a .git/
+// not stupid like my old version
+func gitDirectoriesNew(srcDir string) ([]string, error) {
+	var all []string
+	var trip bool
+	err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			// Handle possible errors, like permission issues
+			fmt.Fprintf(os.Stderr, "error accessing path %q: %v\n", path, err)
+			return err
+		}
+
+		if d.IsDir() {
+			// log.Info("path is dir", path)
+		} else {
+			_, fname := filepath.Split(path)
+			switch fname {
+			case "repos.pb":
+			case "go.work":
+			case "go.work.last":
+			case "go.work.sum":
+			default:
+				// todo: figure out a way to do padding for init()
+				if trip == false {
+					log.Info("WARNING:")
+				}
+				log.Info("WARNING: you have an untracked file outside of any .git repository:", path)
+				trip = true
+			}
+			return nil
+		}
+
+		gitdir := filepath.Join(path, ".git")
+		_, err2 := os.Stat(gitdir)
+		if !os.IsNotExist(err2) {
+			all = append(all, path)
+			return filepath.SkipDir
+		}
+		return nil
+	})
+	//
+	// probably always leave this here forever
+	// this check, along with CheckDirty() makes sure you can safely delete ~/go/src or the go.work directory
+	// because everything is either checked in or deleted. An important thing to know!
+	if trip {
+		log.Info("WARNING:")
+		log.Info("WARNING: there isn't a way to disable this warning yet")
+		log.Info("WARNING: probably this is a good thing however. you don't want to leave files outside of git repos here")
+		log.Info("WARNING: so this warning should probably stay")
+		log.Info("WARNING:")
+		log.Info("WARNING: this also might mean you put these files here because you are actively working on them")
+		log.Info("WARNING: and you don't want to forget about them")
+		log.Info("WARNING:")
+	}
+	return all, err
+}

init.go

@@ -80,8 +80,8 @@ func initFromConfig(cfg *ForgeConfigs) *Forge {
 	f.Patchsets = NewPatchsets()
 
 	// todo: play with these / determine good values based on user's machine
-	f.rillX = 10
+	f.Config.RillX = 10
-	f.rillY = 20
+	f.Config.RillY = 20
 	return f
 }
 
@@ -100,7 +100,6 @@ func (f *Forge) Exit() {
 	f.ConfigSave()
 	if f.Repos != nil {
 		if config.HasChanged("repos") {
-			log.Info("TRYING FILENAME:", f.Config.ReposPB)
 			if err := f.Repos.ConfigSave(f.Config.ReposPB); err != nil {
 				log.Info("forge.Repos.ConfigSave() error", err)
 			}

scanRepoDir.go

@@ -0,0 +1,117 @@
+package forgepb
+
+import (
+	"os"
+
+	"github.com/destel/rill"
+	"go.wit.com/lib/config"
+	"go.wit.com/lib/protobuf/gitpb"
+	"go.wit.com/log"
+)
+
+func reloadCheck(repo *gitpb.Repo) error {
+	if err := repo.ReloadCheck(); err != nil {
+		log.Infof("%s reload() says %v\n", repo.FullPath, err)
+		return err
+	}
+	return nil
+}
+
+func (f *Forge) TestScan() error {
+	f.Repos = gitpb.NewRepos()
+	dirs, err := gitDirectoriesNew(f.Config.ReposDir)
+	if err != nil {
+		return err
+	}
+	for i, fullpath := range dirs {
+		repo, err := gitpb.NewRepo(fullpath)
+		if err != nil {
+			log.Info("ReAdd() error", fullpath, err)
+		}
+		log.Info(i, "worked", repo.FullPath)
+		repo = f.Repos.Append(repo)
+		f.VerifyBranchNames(repo)
+		if f.Config.IsReadOnly(repo.GetGoPath()) {
+			repo.ReadOnly = true
+		}
+		repo.ReloadCheck()
+		if i > 5 {
+			break
+		}
+	}
+	return nil
+}
+
+func (f *Forge) checkNamespace(fullpath string) (*gitpb.Repo, error) {
+	if repo := f.Repos.FindByFullPath(fullpath); repo != nil {
+		return nil, nil
+	}
+	repo, err := gitpb.NewRepo(fullpath)
+	if err != nil {
+		log.Info(fullpath, err)
+		return nil, err
+	}
+	return repo, err
+}
+
+func (f *Forge) ScanRepoDir() error {
+	if r := recover(); r != nil {
+		log.Warn("panic ecovered in", r)
+		// debug.PrintStack()
+		// 	pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		panic(os.Stdout)
+	}
+	dirs, err := gitDirectoriesNew(f.Config.ReposDir)
+	if err != nil {
+		return err
+	}
+
+	stats := f.RillRepos(reloadCheck)
+	for _, stat := range stats {
+		if stat.Err == nil {
+			continue
+		}
+		config.SetChanged("repos", true)
+	}
+
+	newcount, err := f.rillScanDirsNew(dirs)
+	if err != nil {
+		log.Info("go src dir problem. exit for now?", err)
+		return err
+	}
+	if newcount != 0 {
+		log.Info("forge go src scan found", newcount, "repos")
+		config.SetChanged("repos", true)
+	}
+	return err
+}
+
+// rill is awesome. long live rill
+// attempt scan with rill
+func (f *Forge) rillScanDirsNew(fullpaths []string) (int, error) {
+	// Convert a slice of user IDs into a channel
+	ids := rill.FromSlice(fullpaths, nil)
+
+	// Read users from the API.  // Concurrency = 20
+	dirs := rill.Map(ids, int(f.Config.RillX), func(id string) (*gitpb.Repo, error) {
+		return f.checkNamespace(id)
+	})
+
+	var counter int
+	// Activate users.  // Concurrency = 10
+	err := rill.ForEach(dirs, int(f.Config.RillY), func(repo *gitpb.Repo) error {
+		if repo == nil {
+			return nil
+		}
+		repo = f.Repos.Append(repo)
+		f.VerifyBranchNames(repo)
+		if f.Config.IsReadOnly(repo.GetGoPath()) {
+			repo.ReadOnly = true
+		}
+		repo.ReloadCheck()
+		counter += 1
+		return nil
+	})
+
+	return counter, err
+}