jcarr
/
forgepb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

finally a rewrite of the old junky scanner

This commit is contained in:
Jeff Carr 2025-09-13 06:26:55 -05:00
parent 0f895e83e6
commit 27a539b879
6 changed files with 186 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
doDirty.go
View File

@ -23,8 +23,7 @@ func (f *Forge) CheckDirtyQuiet() {
for _, s := range stats {
if s.Err == nil {
} else {
// log.Info("forge SetConfigSave(true)")
f.SetConfigSave(true)
config.SetChanged("repos", true)
changed = true
}
}
@ -46,7 +45,7 @@ func (f *Forge) CheckDirty() *gitpb.Repos {
if s.Err == nil {
} else {
log.Info(i, s.Err)
f.SetConfigSave(true)
config.SetChanged("repos", true)
}
}
@ -74,16 +73,12 @@ func doCheckDirty(repo *gitpb.Repo) error {
// nothing changed
} else {
log.Info("Repo changed to clean", repo.FullPath)
config.SetChanged("repos", true)
return log.Errorf("%s repo changed to clean", repo.FullPath)
// f.SetConfigSave(true)
}
} else {
if repo.CheckDirty() {
log.Info("Repo changed to dirty", repo.FullPath)
config.SetChanged("repos", true)
return log.Errorf("%s repo changed to dirty", repo.FullPath)
// f.SetConfigSave(true)
} else {
// nothing changed
}

2
forgeConfig.proto
View File

@ -62,6 +62,8 @@ message ForgeConfigs { // `autogenpb:mar
string PatchDir = 13; // patch dir
string ForgeURL = 14; // forge URL
string Filename = 15; // filename of the config file
int32 rillX = 16; // used by rill
int32 rillY = 17; // used by rill
}
// this generic message is used by autogen to identify and

8
goSrcFind.go
View File

@ -4,12 +4,6 @@ package forgepb
// If there is a go.work file in your parent, that directory will be returned
// otherwise, return ~/go/src
import (
"fmt"
"os"
"path/filepath"
)
func (f *Forge) GetHome() string {
return f.Config.ReposDir
}
@ -70,6 +64,7 @@ func (f *Forge) goWorkExists() bool {
}
*/
/*
func digup(path string) (string, error) {
for {
workFilePath := filepath.Join(path, "go.work")
@ -88,3 +83,4 @@ func digup(path string) (string, error) {
return "", fmt.Errorf("no go.work file found")
}
*/

164
goSrcScan.go
View File

@ -1,50 +1,14 @@
package forgepb
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/destel/rill"
"go.wit.com/lib/protobuf/gitpb"
"go.wit.com/log"
)
func reloadCheck(repo *gitpb.Repo) error {
if err := repo.ReloadCheck(); err != nil {
log.Infof("%s reload() says %v\n", repo.FullPath, err)
return err
}
return nil
}
func (f *Forge) TestScan() error {
f.Repos = gitpb.NewRepos()
dirs, err := gitDirectoriesNew(f.Config.ReposDir)
if err != nil {
return err
}
for i, fullpath := range dirs {
repo, err := gitpb.NewRepo(fullpath)
if err != nil {
log.Info("ReAdd() error", fullpath, err)
}
log.Info(i, "worked", repo.FullPath)
repo = f.Repos.Append(repo)
f.VerifyBranchNames(repo)
if f.Config.IsReadOnly(repo.GetGoPath()) {
repo.ReadOnly = true
}
repo.ReloadCheck()
if i > 5 {
break
}
}
return nil
}
/*
func (f *Forge) ScanGoSrc() (bool, error) {
dirs, err := gitDirectoriesNew(f.Config.ReposDir)
if err != nil {
@ -56,18 +20,11 @@ func (f *Forge) ScanGoSrc() (bool, error) {
if stat.Err == nil {
continue
}
f.SetConfigSave(true)
/*
dur := stat.End.Sub(stat.Start)
if dur > time.Second {
log.Infof("%s checkRemoteBranches() took a long time (%s) (err=%v)\n", path, shell.FormatDuration(dur), stat.Err)
}
*/
config.SetChanged("repos", true)
}
var gopaths []string
for _, dir := range dirs {
// log.Info("forge.ScanGoSrc()", dir)
if strings.HasPrefix(dir, f.Config.ReposDir) {
gopath := strings.TrimPrefix(dir, f.Config.ReposDir)
gopath = strings.Trim(gopath, "/")
@ -88,7 +45,7 @@ func (f *Forge) ScanGoSrc() (bool, error) {
}
if newcount != 0 {
log.Info("forge go src scan found", newcount, "repos")
f.SetConfigSave(true)
config.SetChanged("repos", true)
}
return true, err
}
@ -103,63 +60,6 @@ func (f *Forge) ScanDir(dir string) *gitpb.Repo {
return repo
}
// doesn't enter the directory any further when it finds a .git/
// not stupid like my old version
func gitDirectoriesNew(srcDir string) ([]string, error) {
var all []string
var trip bool
err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, err error) error {
if err != nil {
// Handle possible errors, like permission issues
fmt.Fprintf(os.Stderr, "error accessing path %q: %v\n", path, err)
return err
}
if d.IsDir() {
// log.Info("path is dir", path)
} else {
_, fname := filepath.Split(path)
switch fname {
case "repos.pb":
case "go.work":
case "go.work.last":
case "go.work.sum":
default:
// todo: figure out a way to do padding for init()
if trip == false {
log.Info("WARNING:")
}
log.Info("WARNING: you have an untracked file outside of any .git repository:", path)
trip = true
}
return nil
}
gitdir := filepath.Join(path, ".git")
_, err2 := os.Stat(gitdir)
if !os.IsNotExist(err2) {
all = append(all, path)
return filepath.SkipDir
}
return nil
})
//
// probably always leave this here forever
// this check, along with CheckDirty() makes sure you can safely delete ~/go/src or the go.work directory
// because everything is either checked in or deleted. An important thing to know!
if trip {
log.Info("WARNING:")
log.Info("WARNING: there isn't a way to disable this warning yet")
log.Info("WARNING: probably this is a good thing however. you don't want to leave files outside of git repos here")
log.Info("WARNING: so this warning should probably stay")
log.Info("WARNING:")
log.Info("WARNING: this also might mean you put these files here because you are actively working on them")
log.Info("WARNING: and you don't want to forget about them")
log.Info("WARNING:")
}
return all, err
}
func gitDirectoriesOld(srcDir string) ([]string, error) {
var all []string
err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
@ -246,3 +146,61 @@ func (f *Forge) ReAdd(repo *gitpb.Repo) (*gitpb.Repo, error) {
}
return repo, err
}
*/
// doesn't enter the directory any further when it finds a .git/
// not stupid like my old version
func gitDirectoriesNew(srcDir string) ([]string, error) {
var all []string
var trip bool
err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, err error) error {
if err != nil {
// Handle possible errors, like permission issues
fmt.Fprintf(os.Stderr, "error accessing path %q: %v\n", path, err)
return err
}
if d.IsDir() {
// log.Info("path is dir", path)
} else {
_, fname := filepath.Split(path)
switch fname {
case "repos.pb":
case "go.work":
case "go.work.last":
case "go.work.sum":
default:
// todo: figure out a way to do padding for init()
if trip == false {
log.Info("WARNING:")
}
log.Info("WARNING: you have an untracked file outside of any .git repository:", path)
trip = true
}
return nil
}
gitdir := filepath.Join(path, ".git")
_, err2 := os.Stat(gitdir)
if !os.IsNotExist(err2) {
all = append(all, path)
return filepath.SkipDir
}
return nil
})
//
// probably always leave this here forever
// this check, along with CheckDirty() makes sure you can safely delete ~/go/src or the go.work directory
// because everything is either checked in or deleted. An important thing to know!
if trip {
log.Info("WARNING:")
log.Info("WARNING: there isn't a way to disable this warning yet")
log.Info("WARNING: probably this is a good thing however. you don't want to leave files outside of git repos here")
log.Info("WARNING: so this warning should probably stay")
log.Info("WARNING:")
log.Info("WARNING: this also might mean you put these files here because you are actively working on them")
log.Info("WARNING: and you don't want to forget about them")
log.Info("WARNING:")
}
return all, err
}

5
init.go
View File

@ -80,8 +80,8 @@ func initFromConfig(cfg *ForgeConfigs) *Forge {
f.Patchsets = NewPatchsets()
// todo: play with these / determine good values based on user's machine
f.rillX = 10
f.rillY = 20
f.Config.RillX = 10
f.Config.RillY = 20
return f
}
@ -100,7 +100,6 @@ func (f *Forge) Exit() {
f.ConfigSave()
if f.Repos != nil {
if config.HasChanged("repos") {
log.Info("TRYING FILENAME:", f.Config.ReposPB)
if err := f.Repos.ConfigSave(f.Config.ReposPB); err != nil {
log.Info("forge.Repos.ConfigSave() error", err)
}

117
scanRepoDir.go Normal file
View File

@ -0,0 +1,117 @@
package forgepb
import (
"os"
"github.com/destel/rill"
"go.wit.com/lib/config"
"go.wit.com/lib/protobuf/gitpb"
"go.wit.com/log"
)
func reloadCheck(repo *gitpb.Repo) error {
if err := repo.ReloadCheck(); err != nil {
log.Infof("%s reload() says %v\n", repo.FullPath, err)
return err
}
return nil
}
func (f *Forge) TestScan() error {
f.Repos = gitpb.NewRepos()
dirs, err := gitDirectoriesNew(f.Config.ReposDir)
if err != nil {
return err
}
for i, fullpath := range dirs {
repo, err := gitpb.NewRepo(fullpath)
if err != nil {
log.Info("ReAdd() error", fullpath, err)
}
log.Info(i, "worked", repo.FullPath)
repo = f.Repos.Append(repo)
f.VerifyBranchNames(repo)
if f.Config.IsReadOnly(repo.GetGoPath()) {
repo.ReadOnly = true
}
repo.ReloadCheck()
if i > 5 {
break
}
}
return nil
}
func (f *Forge) checkNamespace(fullpath string) (*gitpb.Repo, error) {
if repo := f.Repos.FindByFullPath(fullpath); repo != nil {
return nil, nil
}
repo, err := gitpb.NewRepo(fullpath)
if err != nil {
log.Info(fullpath, err)
return nil, err
}
return repo, err
}
func (f *Forge) ScanRepoDir() error {
if r := recover(); r != nil {
log.Warn("panic ecovered in", r)
// debug.PrintStack()
// pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
panic(os.Stdout)
}
dirs, err := gitDirectoriesNew(f.Config.ReposDir)
if err != nil {
return err
}
stats := f.RillRepos(reloadCheck)
for _, stat := range stats {
if stat.Err == nil {
continue
}
config.SetChanged("repos", true)
}
newcount, err := f.rillScanDirsNew(dirs)
if err != nil {
log.Info("go src dir problem. exit for now?", err)
return err
}
if newcount != 0 {
log.Info("forge go src scan found", newcount, "repos")
config.SetChanged("repos", true)
}
return err
}
// rill is awesome. long live rill
// attempt scan with rill
func (f *Forge) rillScanDirsNew(fullpaths []string) (int, error) {
// Convert a slice of user IDs into a channel
ids := rill.FromSlice(fullpaths, nil)
// Read users from the API. // Concurrency = 20
dirs := rill.Map(ids, int(f.Config.RillX), func(id string) (*gitpb.Repo, error) {
return f.checkNamespace(id)
})
var counter int
// Activate users. // Concurrency = 10
err := rill.ForEach(dirs, int(f.Config.RillY), func(repo *gitpb.Repo) error {
if repo == nil {
return nil
}
repo = f.Repos.Append(repo)
f.VerifyBranchNames(repo)
if f.Config.IsReadOnly(repo.GetGoPath()) {
repo.ReadOnly = true
}
repo.ReloadCheck()
counter += 1
return nil
})
return counter, err
}