diff --git a/all.go b/all.go new file mode 100644 index 0000000..fbdafc7 --- /dev/null +++ b/all.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" +) + +// rethink this. do not run on non-master git branches +func doAll() { + if argv.All { + if forge.IsGoWork() { + var warning []string + warning = append(warning, "go-mod-clean --recursive may not work unless you are in ~/go/src") + warning = append(warning, "you can continue anyway, but it hasn't been tested as much.") + simpleStdin(true, warning) + } + var warning []string + warning = append(warning, "go-mod-clean will recreate go.mod and go.sum") + warning = append(warning, "because you have selected --recursive") + warning = append(warning, "this will redo _every_ repo. This is probably fine.") + warning = append(warning, fmt.Sprintf("You have %d total repositories in %s", forge.Repos.Len(), forge.GetGoSrc())) + warning = append(warning, "") + warning = append(warning, "However, this will also, at times, do:") + warning = append(warning, "") + warning = append(warning, "rm -rf ~/go/pkg/") + warning = append(warning, "rm -rf ~/.config/go-build/") + warning = append(warning, "") + warning = append(warning, "Which is also probably fine, but will clear all your build cache and go mod cache") + warning = append(warning, "") + simpleStdin(false, warning) + // purgeGoCaches() + } +} diff --git a/argv.go b/argv.go index 6161076..707791e 100644 --- a/argv.go +++ b/argv.go @@ -7,8 +7,13 @@ package main var argv args type args struct { - Recursive bool `arg:"--recursive" default:"false" help:"clean every repo found in go/src or go.work"` - Auto bool `arg:"--auto" help:"don't approve via STDIN"` + All bool `arg:"--all" default:"false" help:"redo every repo found in go/src or go.work"` + Auto bool `arg:"--auto" help:"don't approve via STDIN"` + Trim bool `arg:"--trim" default:"true" help:"trim entries from go.sum"` + Verbose bool `arg:"--verbose" help:"show more"` + Notes bool `arg:"--metadata" help:"save as git metadata (notes)"` + Restore bool `arg:"--restore" default:"true" help:"restore from git metadata"` + Force bool `arg:"--force" help:"remove the old one"` } func (args) Version() string { @@ -20,6 +25,7 @@ func (a args) Description() string { go-mod-clean will try to verify your go.* files are using the newest package versions * Recreate go.* with 'go mod init' and 'go mod tidy' +* Set your required go in go.mod (default is go1.20 * Check that the most recent master branch versions are used * Try to trim go.sum of non-existent entries ` diff --git a/cleanGoSum.go b/cleanGoSum.go new file mode 100644 index 0000000..76e5037 --- /dev/null +++ b/cleanGoSum.go @@ -0,0 +1,147 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "go.wit.com/lib/protobuf/gitpb" + "go.wit.com/log" +) + +// This will recreate your go.sum and go.mod files + +// checks to see if every 'master' git branch version +// matches the go.sum file +func cleanGoDepsCheckOk(check *gitpb.Repo) error { + var err error = nil + log.Printf("current repo %s go dependancy count: %d", check.GetGoPath(), check.GoDepsLen()) + all := check.GoDeps.SortByGoPath() + for all.Scan() { + depRepo := all.Next() + found := forge.Repos.FindByGoPath(depRepo.GetGoPath()) + if found == nil { + if forge.CheckOverride(depRepo.GetGoPath()) { + // skip this gopath because it's probably broken forever + continue + } + log.Info("not found:", depRepo.GetGoPath()) + err = errors.New("not found: " + depRepo.GetGoPath()) + continue + } + // log.Info("found dep", depRepo.GetGoPath()) + if depRepo.GetVersion() != found.GetMasterVersion() { + check := forge.Repos.FindByGoPath(depRepo.GoPath) + var ends string + if check.CheckDirty() { + ends = "(dirty) " + } + + if forge.Config.IsReadOnly(check.GoPath) { + ends += "(ignoring read-only) " + if argv.Verbose { + log.Printf("%-48s ok error .%s. vs .%s. %s", depRepo.GetGoPath(), + depRepo.GetVersion(), found.GetMasterVersion(), ends) + } + } else { + if forge.CheckOverride(depRepo.GetGoPath()) { + ends += "(override) " + if argv.Verbose { + log.Printf("%-48s ok error .%s. vs .%s. %s", depRepo.GetGoPath(), + depRepo.GetVersion(), found.GetMasterVersion(), ends) + // skip this gopath because it's probably broken forever + } + continue + } else { + log.Printf("%-48s error %10s vs %10s %s", depRepo.GetGoPath(), + depRepo.GetVersion(), found.GetMasterVersion(), ends) + errs := fmt.Sprintf("%s error %s vs %s %s", depRepo.GetGoPath(), + depRepo.GetVersion(), found.GetMasterVersion(), ends) + err = errors.New(errs) + } + } + } + } + return err +} + +func trimGoSum(check *gitpb.Repo) error { + var stuff map[string]string + stuff = make(map[string]string) + + var modver map[string]string + modver = make(map[string]string) + + var good map[string]bool + good = make(map[string]bool) + + if check == nil { + log.Info("boo, check == nil") + return errors.New("*repo == nil") + } + filename := filepath.Join(filepath.Join(check.FullPath, "go.sum")) + data, err := os.ReadFile(filename) + if err != nil { + return err + } + + for _, line := range strings.Split(string(data), "\n") { + parts := strings.Fields(line) + if len(parts) < 3 { + log.Info("WIERD OR BAD:", line) + continue + } + + gopath := parts[0] + version := parts[1] + hash := parts[2] + + if strings.HasSuffix(version, "/go.mod") { + if _, ok := stuff[gopath]; ok { + if argv.Verbose { + log.Info("MATCHED: gopath:", gopath, "version:", version) + } + modver[gopath] = version + " " + hash + good[gopath] = true + } else { + if argv.Verbose { + log.Info("GARBAGE: gopath:", gopath, "version:", version) + } + } + } else { + if argv.Verbose { + log.Info("GOOD : gopath:", gopath, "version:", version) + } + stuff[gopath] = version + " " + hash + } + } + + keys := make([]string, 0, len(stuff)) + for k := range stuff { + keys = append(keys, k) + } + + // rewrite the go.sum file + newfilename := filepath.Join(filepath.Join(check.FullPath, "go.sum")) + newf, err := os.OpenFile(newfilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer newf.Close() + sort.Strings(keys) + for _, gopath := range keys { + if good[gopath] { + fmt.Fprintf(newf, "%s %s\n", gopath, stuff[gopath]) + fmt.Fprintf(newf, "%s %s\n", gopath, modver[gopath]) + check := forge.Repos.FindByGoPath(gopath) + if check == nil { + log.Info("gopath does not really exist:", gopath) + } + } + } + // fmt.Fprintln(newf, "test") + return nil +} diff --git a/main.go b/main.go index 72a0fef..8d6cdf4 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,10 @@ package main import ( + "errors" "fmt" "os" + "path/filepath" "strings" "go.wit.com/dev/alexflint/arg" @@ -26,8 +28,6 @@ func main() { // load the ~/.config/forge/ config // this lets you configure repos you have read/write access too forge = forgepb.Init() - // rescan just in case (?) todo: decide what forge should default too - forge.ScanGoSrc() // figure out what directory we are running in check = findPwdRepo() @@ -35,51 +35,50 @@ func main() { log.Info("this directory isn't in a golang project (not in ~/go/src nor a go.work file)") os.Exit(-1) } - log.Info("starting go-mod-clean for", check.GoPath) - log.Info("go src dir is set to", forge.GetGoSrc()) - if argv.Recursive { - if forge.IsGoWork() { - var warning []string - warning = append(warning, "go-mod-clean --recursive may not work unless you are in ~/go/src") - warning = append(warning, "you can continue anyway, but it hasn't been tested as much.") - simpleStdin(true, warning) + // skip restore if --force + if !argv.Force { + // try to restore from the git metadata + if restoreFromGit(check) { + okExit("go.mod was restored from the git notes") } - var warning []string - warning = append(warning, "go-mod-clean will recreate go.mod and go.sum") - warning = append(warning, "because you have selected --recursive") - warning = append(warning, "this will redo _every_ repo. This is probably fine.") - warning = append(warning, fmt.Sprintf("You have %d total repositories in %s", forge.Repos.Len(), forge.GetGoSrc())) - warning = append(warning, "") - warning = append(warning, "However, this will also, at times, do:") - warning = append(warning, "") - warning = append(warning, "rm -rf ~/go/pkg/") - warning = append(warning, "rm -rf ~/.config/go-build/") - warning = append(warning, "") - warning = append(warning, "Which is also probably fine, but will clear all your build cache and go mod cache") - warning = append(warning, "") - simpleStdin(false, warning) - // purgeGoCaches() - } else { - simpleStdin(true, []string{"go-mod-clean will recreate go.mod and go.sum"}) + } + + if check.GetMasterBranchName() != check.GetCurrentBranchName() { + log.Info("") + log.Info("You can only run go-mod-clean on a git master branch.") + log.Info("Publishing go.mod & go.sum files must come from from git version tag") + log.Info("Anything else doesn't make sense.") + log.Info("") + badExit(errors.New("not git master branch")) } // re-create go.sum and go.mod - if _, err := check.RedoGoMod(); err != nil { + if _, err := redoGoMod(check); err != nil { badExit(err) } - // try to trim junk - if err := forge.TrimGoSum(check); err != nil { - badExit(err) + if argv.Trim { + // try to trim junk + if err := trimGoSum(check); err != nil { + badExit(err) + } } // check go.sum file - if err := forge.CleanGoDepsCheckOk(check); err != nil { + if err := cleanGoDepsCheckOk(check); err != nil { log.Info("forge.FinalGoDepsCheck() failed. boo. :", check.GoPath) badExit(err) } + // put the files in the notes section in git + // this way, git commits are not messed up + // with this autogenerated code + if err := saveAsMetadata(check); err != nil { + log.Info("save go.mod as git metadata failed", check.GoPath, err) + badExit(err) + } + log.Info("forge.FinalGoDepsCheck() worked :", check.GoPath) okExit(check.GoPath + " go.sum seems clean") } @@ -103,12 +102,70 @@ func findPwdRepo() *gitpb.Repo { } func okExit(thing string) { + log.DaemonMode(true) log.Info(thing, "ok") - log.Info("Finished go-mod-clean on", check.GetGoPath(), "ok") + // log.Info("Finished go-mod-clean on", check.GetGoPath(), "ok") os.Exit(0) } func badExit(err error) { + log.DaemonMode(true) log.Info("go-mod-clean failed: ", err, forge.GetGoSrc()) os.Exit(-1) } + +// todo: do this the right way in git +func saveAsMetadata(repo *gitpb.Repo) error { + cname := check.GetCurrentBranchName() + cmd := []string{"git", "notes", "remove", cname} + if err := check.StrictRun(cmd); err != nil { + return err + } + if check.GoPrimitive { + cmd = []string{"git", "notes", "add", "-F", "go.mod", cname} + if err := check.StrictRun(cmd); err != nil { + return err + } + } else { + cmd = []string{"git", "notes", "add", "-F", "go.mod", cname} + if err := check.StrictRun(cmd); err != nil { + return err + } + cmd = []string{"git", "notes", "append", "-m", "GOSUM:", cname} + if err := check.StrictRun(cmd); err != nil { + return err + } + cmd = []string{"git", "notes", "append", "-F", "go.sum", cname} + if err := check.StrictRun(cmd); err != nil { + return err + } + } + return nil +} + +func restoreFromGit(repo *gitpb.Repo) bool { + result := repo.Run([]string{"git", "notes", "show"}) + if result.Exit != 0 { + return false + } + if result.Error != nil { + return false + } + if len(result.Stdout) == 0 { + return false + } + + all := strings.Join(result.Stdout, "\n") + parts := strings.Split(all, "GOSUM:") + + gomod := filepath.Join(filepath.Join(check.FullPath, "go.mod")) + newf, _ := os.OpenFile(gomod, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + fmt.Fprint(newf, strings.TrimSpace(parts[0])) + + if len(parts) == 2 { + gosum := filepath.Join(filepath.Join(check.FullPath, "go.sum")) + newf, _ := os.OpenFile(gosum, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + fmt.Fprint(newf, strings.TrimSpace(parts[1])) + } + return true +} diff --git a/redoGoMod.go b/redoGoMod.go new file mode 100644 index 0000000..f25a3b4 --- /dev/null +++ b/redoGoMod.go @@ -0,0 +1,85 @@ +package main + +// recreates the go.mod and go.sum files + +import ( + "errors" + "os" + + "go.wit.com/lib/protobuf/gitpb" + "go.wit.com/log" +) + +// remove every go.mod and go.sum +// testing to see where this stuff is coming from +func eraseGoMod(repo *gitpb.Repo) { + // unset the go development ENV var to generate release files + if err := repo.StrictRun([]string{"rm", "-f", "go.mod", "go.sum"}); err != nil { + log.Warn(repo.GoPath, "rm go.mod go.sum failed", err) + } +} + +// sets the required golang version in go.mod +func setGoVersion(repo *gitpb.Repo, version string) error { + // most things should build with golang after 1.20 + if err := repo.StrictRun([]string{"go", "mod", "edit", "-go=" + version}); err != nil { + log.Warn(repo.GoPath, "go mod edit failed", err) + return err + } + return nil +} + +// wrapper around 'go mod init' and 'go mod tidy' +func redoGoMod(repo *gitpb.Repo) (bool, error) { + // unset the go development ENV var to generate release files + os.Unsetenv("GO111MODULE") + if err := repo.StrictRun([]string{"rm", "-f", "go.mod", "go.sum"}); err != nil { + log.Warn("rm go.mod go.sum failed", err) + return false, err + } + if err := repo.StrictRun([]string{"go", "mod", "init", repo.GoPath}); err != nil { + log.Warn("go mod init failed", err) + return false, err + } + if err := repo.StrictRun([]string{"go", "mod", "tidy"}); err != nil { + log.Warn("go mod tidy failed", err) + return false, err + } + + // most things should build with golang after 1.20 // todo: allow this to be set somewhere + if err := setGoVersion(repo, "1.20"); err != nil { + log.Warn(repo.GoPath, "go mod edit failed", err) + return false, err + } + + repo.GoDeps = nil + repo.GoPrimitive = false + + // if there is not a go.sum file, it better be a primitive golang project + if !repo.Exists("go.sum") { + // this should never happen + return false, errors.New("MakeRedomod() logic failed") + + ok, err := repo.IsPrimitive() + if err != nil { + // this means this repo does not depend on any other package + log.Info("PRIMATIVE repo error:", repo.GoPath, "err =", err) + return false, err + } + + if ok { + // this means the repo is primitive so there is no go.sum + repo.GoPrimitive = true + return true, nil + } + } + + repo.GoDeps = new(gitpb.GoDeps) + if !repo.Exists("go.sum") { + // this should never happen + return false, errors.New("MakeRedomod() logic failed") + } + + // return the attempt to parse go.sum + return repo.ParseGoSum() +}