From a822e1e4f072fbd735efd328e06fe2bf48a2d660 Mon Sep 17 00:00:00 2001 From: Jeff Carr <jcarr@wit.com> Date: Sat, 18 Jan 2025 23:26:19 -0600 Subject: [PATCH] switch some more things to use rill --- goSrcScan.go | 37 ------------------------- rill.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 37 deletions(-) diff --git a/goSrcScan.go b/goSrcScan.go index 357fed3..a934b15 100644 --- a/goSrcScan.go +++ b/goSrcScan.go @@ -201,40 +201,3 @@ func (f *Forge) RillRedoGoMod() int { return counter } - -// x is the size of the queued up pool (shouldn't matter here for this I think) -// y is how many simultanous functions will run -// todo: tune and compute x,y by # of CPUs and disk io -// todo: store x,y in forge config ? (or compute them. notsure) -func (f *Forge) RillFuncError(x int, y int, rillf func(*gitpb.Repo) error) int { - var all []*gitpb.Repo - tmp := f.Repos.All() - for tmp.Scan() { - repo := tmp.Next() - if !repo.IsValidDir() { - log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath()) - continue - } - all = append(all, repo) - } - // Convert a slice of user IDs into a channel - ids := rill.FromSlice(all, nil) - - var counter int - // Read users from the API. - // Concurrency = 20 - dirs := rill.Map(ids, x, func(id *gitpb.Repo) (*gitpb.Repo, error) { - return id, nil - }) - - err := rill.ForEach(dirs, y, func(repo *gitpb.Repo) error { - counter += 1 - return rillf(repo) - }) - - if err != nil { - log.Info("rill.ForEach() error:", err) - } - - return counter -} diff --git a/rill.go b/rill.go index 27af0f2..b07128f 100644 --- a/rill.go +++ b/rill.go @@ -64,3 +64,81 @@ func (f *Forge) updateRepo(repo *gitpb.Repo) error { } return nil } + +var RillX int = 30 +var RillY int = 10 + +// x is the size of the queued up pool (shouldn't matter here for this I think) +// y is how many simultanous functions will run +// todo: tune and compute x,y by # of CPUs and disk io +// todo: store x,y in forge config ? (or compute them. notsure) +func (f *Forge) RillReload() int { + var all []*gitpb.Repo + tmp := f.Repos.All() + for tmp.Scan() { + repo := tmp.Next() + if !repo.IsValidDir() { + log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath()) + continue + } + all = append(all, repo) + } + // Convert a slice of user IDs into a channel + ids := rill.FromSlice(all, nil) + + var counter int + // Read users from the API. + // Concurrency = 20 + dirs := rill.Map(ids, RillX, func(repo *gitpb.Repo) (*gitpb.Repo, error) { + return repo, nil + }) + + rill.ForEach(dirs, RillY, func(repo *gitpb.Repo) error { + if !repo.DidRepoChange() { + return nil + } + f.configSave = true + repo.Reload() + counter += 1 + return nil + }) + + return counter +} + +// x is the size of the queued up pool (shouldn't matter here for this I think) +// y is how many simultanous functions will run +// todo: tune and compute x,y by # of CPUs and disk io +// todo: store x,y in forge config ? (or compute them. notsure) +func (f *Forge) RillFuncError(rillf func(*gitpb.Repo) error) int { + var all []*gitpb.Repo + tmp := f.Repos.All() + for tmp.Scan() { + repo := tmp.Next() + if !repo.IsValidDir() { + log.Printf("%s %-50s", "got an invalid repo in forgepb.RillFuncError()", repo.GetGoPath()) + continue + } + all = append(all, repo) + } + // Convert a slice of user IDs into a channel + ids := rill.FromSlice(all, nil) + + var counter int + // Read users from the API. + // Concurrency = 20 + dirs := rill.Map(ids, RillX, func(id *gitpb.Repo) (*gitpb.Repo, error) { + return id, nil + }) + + err := rill.ForEach(dirs, RillY, func(repo *gitpb.Repo) error { + counter += 1 + return rillf(repo) + }) + + if err != nil { + log.Info("rill.ForEach() error:", err) + } + + return counter +}