jcarr
/
forgepb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

standard rill() with stats

This commit is contained in:
Jeff Carr 2025-08-31 15:54:32 -05:00
parent 5147aa73de
commit 7e2caa9290
2 changed files with 55 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cleanGoSum.go
View File

@ -49,20 +49,20 @@ func (f *Forge) CleanGoDepsCheckOk(check *gitpb.Repo) error {
if f.Config.IsReadOnly(check.GetGoPath()) {
ends += "(ignoring read-only) "
if cleanVerbose {
log.Printf("%-48s ok error .%s. vs .%s. %s", depRepo.GetGoPath(),
log.Printf("%-48s ok error .%s. vs .%s. %s\n", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
}
} else {
if f.CheckOverride(depRepo.GetGoPath()) {
ends += "(override) "
if cleanVerbose {
log.Printf("%-48s ok error .%s. vs .%s. %s", depRepo.GetGoPath(),
log.Printf("%-48s ok error .%s. vs .%s. %s\n", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
// skip this gopath because it's probably broken forever
}
continue
} else {
log.Printf("%-48s error %10s vs %10s %s", depRepo.GetGoPath(),
log.Printf("%-48s error %10s vs %10s %s\n", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
errs := fmt.Sprintf("%s error %s vs %s %s", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)

92
rill.go
View File

@ -79,7 +79,7 @@ func (f *Forge) RillReload() int {
var all []*gitpb.Repo
for repo := range f.Repos.IterAll() {
if !repo.IsValidDir() {
log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath())
log.Printf("%s %-50s", "got an invalid repo in forgepb.RillReload()", repo.GetGoPath())
continue
}
all = append(all, repo)
@ -111,47 +111,63 @@ func (f *Forge) RillReload() int {
// y is how many simultanous functions will run
// todo: tune and compute x,y by # of CPUs and disk io
// todo: store x,y in forge config ? (or compute them. notsure)
func (f *Forge) RillFuncError(rillf func(*gitpb.Repo) error) int {
var all []*gitpb.Repo
for repo := range f.Repos.IterAll() {
if !repo.IsValidDir() {
log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath())
continue
func (f *Forge) RillFuncError(rillf func(*gitpb.Repo) error) map[string]*RillStats {
return f.RillRepos(rillf)
/*
var all []*gitpb.Repo
var stats map[string]*RillStats
stats = make(map[string]*RillStats)
for repo := range f.Repos.IterAll() {
if !repo.IsValidDir() {
log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath())
continue
}
all = append(all, repo)
}
all = append(all, repo)
}
// Convert a slice of user IDs into a channel
ids := rill.FromSlice(all, nil)
// Convert a slice of user IDs into a channel
ids := rill.FromSlice(all, nil)
var counter int
var watch int = 10
var meMu sync.Mutex
var counter int
var watch int = 10
var meMu sync.Mutex
// Read users from the API.
// Concurrency = 20
dirs := rill.Map(ids, RillX, func(id *gitpb.Repo) (*gitpb.Repo, error) {
return id, nil
})
// Read users from the API.
// Concurrency = 20
dirs := rill.Map(ids, RillX, func(id *gitpb.Repo) (*gitpb.Repo, error) {
return id, nil
})
err := rill.ForEach(dirs, RillY, func(repo *gitpb.Repo) error {
meMu.Lock()
counter += 1
if counter > watch {
// log.Info("Processed", watch, "repos") // this doesn't work
watch += 50
err := rill.ForEach(dirs, RillY, func(repo *gitpb.Repo) error {
meMu.Lock()
counter += 1
if counter > watch {
// log.Info("Processed", watch, "repos") // this doesn't work
watch += 50
}
meMu.Unlock()
rillSetStartTime(stats, repo.GetFullPath())
err := rillf(repo)
if err != nil {
rillSetError(stats, repo.GetFullPath(), err)
}
rillSetEndTime(stats, repo.GetFullPath())
return err
})
if err != nil {
log.Info("rill.ForEach() error:", err)
}
meMu.Unlock()
return rillf(repo)
})
if err != nil {
log.Info("rill.ForEach() error:", err)
}
return counter
return stats
*/
}
func (f *Forge) ConfigRill(rillX int, rillY int) {
f.rillX = rillX
f.rillY = rillY
log.Infof("Setting rill values to %d,%d\n", f.rillX, f.rillY)
}
type RillStats struct {
@ -169,15 +185,12 @@ var rillMu sync.Mutex
func (f *Forge) RillRepos(rillf func(*gitpb.Repo) error) map[string]*RillStats {
var all []*gitpb.Repo
var allerr map[*gitpb.Repo]error
allerr = make(map[*gitpb.Repo]error)
var stats map[string]*RillStats
stats = make(map[string]*RillStats)
for repo := range f.Repos.IterAll() {
if !repo.IsValidDir() {
log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath())
log.Printf("got an invalid repo in forgepb.RillRepos() %-50s\n", repo.GetGoPath())
continue
}
all = append(all, repo)
@ -195,6 +208,7 @@ func (f *Forge) RillRepos(rillf func(*gitpb.Repo) error) map[string]*RillStats {
})
rill.ForEach(dirs, f.rillY, func(repo *gitpb.Repo) error {
// todo: make this a goroutine to show stats to the user
rillMu.Lock()
counter += 1
if counter > watch {
@ -202,11 +216,9 @@ func (f *Forge) RillRepos(rillf func(*gitpb.Repo) error) map[string]*RillStats {
watch += 50
}
rillMu.Unlock()
rillSetStartTime(stats, repo.GetFullPath())
if err := rillf(repo); err != nil {
rillMu.Lock()
allerr[repo] = err
rillMu.Unlock()
rillSetError(stats, repo.GetFullPath(), err)
}
rillSetEndTime(stats, repo.GetFullPath())