From 27a539b879fd56bbd42d5870c7a2a03979acceba Mon Sep 17 00:00:00 2001 From: Jeff Carr Date: Sat, 13 Sep 2025 06:26:55 -0500 Subject: [PATCH] finally a rewrite of the old junky scanner --- doDirty.go | 9 +-- forgeConfig.proto | 2 + goSrcFind.go | 8 +-- goSrcScan.go | 164 +++++++++++++++++----------------------------- init.go | 5 +- scanRepoDir.go | 117 +++++++++++++++++++++++++++++++++ 6 files changed, 186 insertions(+), 119 deletions(-) create mode 100644 scanRepoDir.go diff --git a/doDirty.go b/doDirty.go index 2dd4ba4..f8d5ec2 100644 --- a/doDirty.go +++ b/doDirty.go @@ -23,8 +23,7 @@ func (f *Forge) CheckDirtyQuiet() { for _, s := range stats { if s.Err == nil { } else { - // log.Info("forge SetConfigSave(true)") - f.SetConfigSave(true) + config.SetChanged("repos", true) changed = true } } @@ -46,7 +45,7 @@ func (f *Forge) CheckDirty() *gitpb.Repos { if s.Err == nil { } else { log.Info(i, s.Err) - f.SetConfigSave(true) + config.SetChanged("repos", true) } } @@ -74,16 +73,12 @@ func doCheckDirty(repo *gitpb.Repo) error { // nothing changed } else { log.Info("Repo changed to clean", repo.FullPath) - config.SetChanged("repos", true) return log.Errorf("%s repo changed to clean", repo.FullPath) - // f.SetConfigSave(true) } } else { if repo.CheckDirty() { log.Info("Repo changed to dirty", repo.FullPath) - config.SetChanged("repos", true) return log.Errorf("%s repo changed to dirty", repo.FullPath) - // f.SetConfigSave(true) } else { // nothing changed } diff --git a/forgeConfig.proto b/forgeConfig.proto index a002214..14ef1ef 100644 --- a/forgeConfig.proto +++ b/forgeConfig.proto @@ -62,6 +62,8 @@ message ForgeConfigs { // `autogenpb:mar string PatchDir = 13; // patch dir string ForgeURL = 14; // forge URL string Filename = 15; // filename of the config file + int32 rillX = 16; // used by rill + int32 rillY = 17; // used by rill } // this generic message is used by autogen to identify and diff --git a/goSrcFind.go b/goSrcFind.go index 02b7659..cef82bb 100644 --- a/goSrcFind.go +++ b/goSrcFind.go @@ -4,12 +4,6 @@ package forgepb // If there is a go.work file in your parent, that directory will be returned // otherwise, return ~/go/src -import ( - "fmt" - "os" - "path/filepath" -) - func (f *Forge) GetHome() string { return f.Config.ReposDir } @@ -70,6 +64,7 @@ func (f *Forge) goWorkExists() bool { } */ +/* func digup(path string) (string, error) { for { workFilePath := filepath.Join(path, "go.work") @@ -88,3 +83,4 @@ func digup(path string) (string, error) { return "", fmt.Errorf("no go.work file found") } +*/ diff --git a/goSrcScan.go b/goSrcScan.go index 4597d0a..e7c45c9 100644 --- a/goSrcScan.go +++ b/goSrcScan.go @@ -1,50 +1,14 @@ package forgepb import ( - "errors" "fmt" "os" "path/filepath" - "strings" - "github.com/destel/rill" - "go.wit.com/lib/protobuf/gitpb" "go.wit.com/log" ) -func reloadCheck(repo *gitpb.Repo) error { - if err := repo.ReloadCheck(); err != nil { - log.Infof("%s reload() says %v\n", repo.FullPath, err) - return err - } - return nil -} - -func (f *Forge) TestScan() error { - f.Repos = gitpb.NewRepos() - dirs, err := gitDirectoriesNew(f.Config.ReposDir) - if err != nil { - return err - } - for i, fullpath := range dirs { - repo, err := gitpb.NewRepo(fullpath) - if err != nil { - log.Info("ReAdd() error", fullpath, err) - } - log.Info(i, "worked", repo.FullPath) - repo = f.Repos.Append(repo) - f.VerifyBranchNames(repo) - if f.Config.IsReadOnly(repo.GetGoPath()) { - repo.ReadOnly = true - } - repo.ReloadCheck() - if i > 5 { - break - } - } - return nil -} - +/* func (f *Forge) ScanGoSrc() (bool, error) { dirs, err := gitDirectoriesNew(f.Config.ReposDir) if err != nil { @@ -56,18 +20,11 @@ func (f *Forge) ScanGoSrc() (bool, error) { if stat.Err == nil { continue } - f.SetConfigSave(true) - /* - dur := stat.End.Sub(stat.Start) - if dur > time.Second { - log.Infof("%s checkRemoteBranches() took a long time (%s) (err=%v)\n", path, shell.FormatDuration(dur), stat.Err) - } - */ + config.SetChanged("repos", true) } var gopaths []string for _, dir := range dirs { - // log.Info("forge.ScanGoSrc()", dir) if strings.HasPrefix(dir, f.Config.ReposDir) { gopath := strings.TrimPrefix(dir, f.Config.ReposDir) gopath = strings.Trim(gopath, "/") @@ -88,7 +45,7 @@ func (f *Forge) ScanGoSrc() (bool, error) { } if newcount != 0 { log.Info("forge go src scan found", newcount, "repos") - f.SetConfigSave(true) + config.SetChanged("repos", true) } return true, err } @@ -103,63 +60,6 @@ func (f *Forge) ScanDir(dir string) *gitpb.Repo { return repo } -// doesn't enter the directory any further when it finds a .git/ -// not stupid like my old version -func gitDirectoriesNew(srcDir string) ([]string, error) { - var all []string - var trip bool - err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - // Handle possible errors, like permission issues - fmt.Fprintf(os.Stderr, "error accessing path %q: %v\n", path, err) - return err - } - - if d.IsDir() { - // log.Info("path is dir", path) - } else { - _, fname := filepath.Split(path) - switch fname { - case "repos.pb": - case "go.work": - case "go.work.last": - case "go.work.sum": - default: - // todo: figure out a way to do padding for init() - if trip == false { - log.Info("WARNING:") - } - log.Info("WARNING: you have an untracked file outside of any .git repository:", path) - trip = true - } - return nil - } - - gitdir := filepath.Join(path, ".git") - _, err2 := os.Stat(gitdir) - if !os.IsNotExist(err2) { - all = append(all, path) - return filepath.SkipDir - } - return nil - }) - // - // probably always leave this here forever - // this check, along with CheckDirty() makes sure you can safely delete ~/go/src or the go.work directory - // because everything is either checked in or deleted. An important thing to know! - if trip { - log.Info("WARNING:") - log.Info("WARNING: there isn't a way to disable this warning yet") - log.Info("WARNING: probably this is a good thing however. you don't want to leave files outside of git repos here") - log.Info("WARNING: so this warning should probably stay") - log.Info("WARNING:") - log.Info("WARNING: this also might mean you put these files here because you are actively working on them") - log.Info("WARNING: and you don't want to forget about them") - log.Info("WARNING:") - } - return all, err -} - func gitDirectoriesOld(srcDir string) ([]string, error) { var all []string err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { @@ -246,3 +146,61 @@ func (f *Forge) ReAdd(repo *gitpb.Repo) (*gitpb.Repo, error) { } return repo, err } +*/ + +// doesn't enter the directory any further when it finds a .git/ +// not stupid like my old version +func gitDirectoriesNew(srcDir string) ([]string, error) { + var all []string + var trip bool + err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + // Handle possible errors, like permission issues + fmt.Fprintf(os.Stderr, "error accessing path %q: %v\n", path, err) + return err + } + + if d.IsDir() { + // log.Info("path is dir", path) + } else { + _, fname := filepath.Split(path) + switch fname { + case "repos.pb": + case "go.work": + case "go.work.last": + case "go.work.sum": + default: + // todo: figure out a way to do padding for init() + if trip == false { + log.Info("WARNING:") + } + log.Info("WARNING: you have an untracked file outside of any .git repository:", path) + trip = true + } + return nil + } + + gitdir := filepath.Join(path, ".git") + _, err2 := os.Stat(gitdir) + if !os.IsNotExist(err2) { + all = append(all, path) + return filepath.SkipDir + } + return nil + }) + // + // probably always leave this here forever + // this check, along with CheckDirty() makes sure you can safely delete ~/go/src or the go.work directory + // because everything is either checked in or deleted. An important thing to know! + if trip { + log.Info("WARNING:") + log.Info("WARNING: there isn't a way to disable this warning yet") + log.Info("WARNING: probably this is a good thing however. you don't want to leave files outside of git repos here") + log.Info("WARNING: so this warning should probably stay") + log.Info("WARNING:") + log.Info("WARNING: this also might mean you put these files here because you are actively working on them") + log.Info("WARNING: and you don't want to forget about them") + log.Info("WARNING:") + } + return all, err +} diff --git a/init.go b/init.go index f14b658..092a5e5 100644 --- a/init.go +++ b/init.go @@ -80,8 +80,8 @@ func initFromConfig(cfg *ForgeConfigs) *Forge { f.Patchsets = NewPatchsets() // todo: play with these / determine good values based on user's machine - f.rillX = 10 - f.rillY = 20 + f.Config.RillX = 10 + f.Config.RillY = 20 return f } @@ -100,7 +100,6 @@ func (f *Forge) Exit() { f.ConfigSave() if f.Repos != nil { if config.HasChanged("repos") { - log.Info("TRYING FILENAME:", f.Config.ReposPB) if err := f.Repos.ConfigSave(f.Config.ReposPB); err != nil { log.Info("forge.Repos.ConfigSave() error", err) } diff --git a/scanRepoDir.go b/scanRepoDir.go new file mode 100644 index 0000000..19de1b0 --- /dev/null +++ b/scanRepoDir.go @@ -0,0 +1,117 @@ +package forgepb + +import ( + "os" + + "github.com/destel/rill" + "go.wit.com/lib/config" + "go.wit.com/lib/protobuf/gitpb" + "go.wit.com/log" +) + +func reloadCheck(repo *gitpb.Repo) error { + if err := repo.ReloadCheck(); err != nil { + log.Infof("%s reload() says %v\n", repo.FullPath, err) + return err + } + return nil +} + +func (f *Forge) TestScan() error { + f.Repos = gitpb.NewRepos() + dirs, err := gitDirectoriesNew(f.Config.ReposDir) + if err != nil { + return err + } + for i, fullpath := range dirs { + repo, err := gitpb.NewRepo(fullpath) + if err != nil { + log.Info("ReAdd() error", fullpath, err) + } + log.Info(i, "worked", repo.FullPath) + repo = f.Repos.Append(repo) + f.VerifyBranchNames(repo) + if f.Config.IsReadOnly(repo.GetGoPath()) { + repo.ReadOnly = true + } + repo.ReloadCheck() + if i > 5 { + break + } + } + return nil +} + +func (f *Forge) checkNamespace(fullpath string) (*gitpb.Repo, error) { + if repo := f.Repos.FindByFullPath(fullpath); repo != nil { + return nil, nil + } + repo, err := gitpb.NewRepo(fullpath) + if err != nil { + log.Info(fullpath, err) + return nil, err + } + return repo, err +} + +func (f *Forge) ScanRepoDir() error { + if r := recover(); r != nil { + log.Warn("panic ecovered in", r) + // debug.PrintStack() + // pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + panic(os.Stdout) + } + dirs, err := gitDirectoriesNew(f.Config.ReposDir) + if err != nil { + return err + } + + stats := f.RillRepos(reloadCheck) + for _, stat := range stats { + if stat.Err == nil { + continue + } + config.SetChanged("repos", true) + } + + newcount, err := f.rillScanDirsNew(dirs) + if err != nil { + log.Info("go src dir problem. exit for now?", err) + return err + } + if newcount != 0 { + log.Info("forge go src scan found", newcount, "repos") + config.SetChanged("repos", true) + } + return err +} + +// rill is awesome. long live rill +// attempt scan with rill +func (f *Forge) rillScanDirsNew(fullpaths []string) (int, error) { + // Convert a slice of user IDs into a channel + ids := rill.FromSlice(fullpaths, nil) + + // Read users from the API. // Concurrency = 20 + dirs := rill.Map(ids, int(f.Config.RillX), func(id string) (*gitpb.Repo, error) { + return f.checkNamespace(id) + }) + + var counter int + // Activate users. // Concurrency = 10 + err := rill.ForEach(dirs, int(f.Config.RillY), func(repo *gitpb.Repo) error { + if repo == nil { + return nil + } + repo = f.Repos.Append(repo) + f.VerifyBranchNames(repo) + if f.Config.IsReadOnly(repo.GetGoPath()) { + repo.ReadOnly = true + } + repo.ReloadCheck() + counter += 1 + return nil + }) + + return counter, err +}