jcarr
/
go-mod-clean
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

save and restore from git notes

This commit is contained in:
Jeff Carr 2024-12-13 02:21:25 -06:00
parent c3e3dfd209
commit 214865f134
5 changed files with 362 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
all.go Normal file
View File

@ -0,0 +1,32 @@
package main
import (
"fmt"
)
// rethink this. do not run on non-master git branches
func doAll() {
if argv.All {
if forge.IsGoWork() {
var warning []string
warning = append(warning, "go-mod-clean --recursive may not work unless you are in ~/go/src")
warning = append(warning, "you can continue anyway, but it hasn't been tested as much.")
simpleStdin(true, warning)
}
var warning []string
warning = append(warning, "go-mod-clean will recreate go.mod and go.sum")
warning = append(warning, "because you have selected --recursive")
warning = append(warning, "this will redo _every_ repo. This is probably fine.")
warning = append(warning, fmt.Sprintf("You have %d total repositories in %s", forge.Repos.Len(), forge.GetGoSrc()))
warning = append(warning, "")
warning = append(warning, "However, this will also, at times, do:")
warning = append(warning, "")
warning = append(warning, "rm -rf ~/go/pkg/")
warning = append(warning, "rm -rf ~/.config/go-build/")
warning = append(warning, "")
warning = append(warning, "Which is also probably fine, but will clear all your build cache and go mod cache")
warning = append(warning, "")
simpleStdin(false, warning)
// purgeGoCaches()
}
}

10
argv.go
View File

@ -7,8 +7,13 @@ package main
var argv args
type args struct {
Recursive bool `arg:"--recursive" default:"false" help:"clean every repo found in go/src or go.work"`
Auto bool `arg:"--auto" help:"don't approve via STDIN"`
All bool `arg:"--all" default:"false" help:"redo every repo found in go/src or go.work"`
Auto bool `arg:"--auto" help:"don't approve via STDIN"`
Trim bool `arg:"--trim" default:"true" help:"trim entries from go.sum"`
Verbose bool `arg:"--verbose" help:"show more"`
Notes bool `arg:"--metadata" help:"save as git metadata (notes)"`
Restore bool `arg:"--restore" default:"true" help:"restore from git metadata"`
Force bool `arg:"--force" help:"remove the old one"`
}
func (args) Version() string {
@ -20,6 +25,7 @@ func (a args) Description() string {
go-mod-clean will try to verify your go.* files are using the newest package versions
* Recreate go.* with 'go mod init' and 'go mod tidy'
* Set your required go in go.mod (default is go1.20
* Check that the most recent master branch versions are used
* Try to trim go.sum of non-existent entries
`

147
cleanGoSum.go Normal file
View File

@ -0,0 +1,147 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"go.wit.com/lib/protobuf/gitpb"
"go.wit.com/log"
)
// This will recreate your go.sum and go.mod files
// checks to see if every 'master' git branch version
// matches the go.sum file
func cleanGoDepsCheckOk(check *gitpb.Repo) error {
var err error = nil
log.Printf("current repo %s go dependancy count: %d", check.GetGoPath(), check.GoDepsLen())
all := check.GoDeps.SortByGoPath()
for all.Scan() {
depRepo := all.Next()
found := forge.Repos.FindByGoPath(depRepo.GetGoPath())
if found == nil {
if forge.CheckOverride(depRepo.GetGoPath()) {
// skip this gopath because it's probably broken forever
continue
}
log.Info("not found:", depRepo.GetGoPath())
err = errors.New("not found: " + depRepo.GetGoPath())
continue
}
// log.Info("found dep", depRepo.GetGoPath())
if depRepo.GetVersion() != found.GetMasterVersion() {
check := forge.Repos.FindByGoPath(depRepo.GoPath)
var ends string
if check.CheckDirty() {
ends = "(dirty) "
}
if forge.Config.IsReadOnly(check.GoPath) {
ends += "(ignoring read-only) "
if argv.Verbose {
log.Printf("%-48s ok error .%s. vs .%s. %s", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
}
} else {
if forge.CheckOverride(depRepo.GetGoPath()) {
ends += "(override) "
if argv.Verbose {
log.Printf("%-48s ok error .%s. vs .%s. %s", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
// skip this gopath because it's probably broken forever
}
continue
} else {
log.Printf("%-48s error %10s vs %10s %s", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
errs := fmt.Sprintf("%s error %s vs %s %s", depRepo.GetGoPath(),
depRepo.GetVersion(), found.GetMasterVersion(), ends)
err = errors.New(errs)
}
}
}
}
return err
}
func trimGoSum(check *gitpb.Repo) error {
var stuff map[string]string
stuff = make(map[string]string)
var modver map[string]string
modver = make(map[string]string)
var good map[string]bool
good = make(map[string]bool)
if check == nil {
log.Info("boo, check == nil")
return errors.New("*repo == nil")
}
filename := filepath.Join(filepath.Join(check.FullPath, "go.sum"))
data, err := os.ReadFile(filename)
if err != nil {
return err
}
for _, line := range strings.Split(string(data), "\n") {
parts := strings.Fields(line)
if len(parts) < 3 {
log.Info("WIERD OR BAD:", line)
continue
}
gopath := parts[0]
version := parts[1]
hash := parts[2]
if strings.HasSuffix(version, "/go.mod") {
if _, ok := stuff[gopath]; ok {
if argv.Verbose {
log.Info("MATCHED: gopath:", gopath, "version:", version)
}
modver[gopath] = version + " " + hash
good[gopath] = true
} else {
if argv.Verbose {
log.Info("GARBAGE: gopath:", gopath, "version:", version)
}
}
} else {
if argv.Verbose {
log.Info("GOOD : gopath:", gopath, "version:", version)
}
stuff[gopath] = version + " " + hash
}
}
keys := make([]string, 0, len(stuff))
for k := range stuff {
keys = append(keys, k)
}
// rewrite the go.sum file
newfilename := filepath.Join(filepath.Join(check.FullPath, "go.sum"))
newf, err := os.OpenFile(newfilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer newf.Close()
sort.Strings(keys)
for _, gopath := range keys {
if good[gopath] {
fmt.Fprintf(newf, "%s %s\n", gopath, stuff[gopath])
fmt.Fprintf(newf, "%s %s\n", gopath, modver[gopath])
check := forge.Repos.FindByGoPath(gopath)
if check == nil {
log.Info("gopath does not really exist:", gopath)
}
}
}
// fmt.Fprintln(newf, "test")
return nil
}

123
main.go
View File

@ -1,8 +1,10 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"go.wit.com/dev/alexflint/arg"
@ -26,8 +28,6 @@ func main() {
// load the ~/.config/forge/ config
// this lets you configure repos you have read/write access too
forge = forgepb.Init()
// rescan just in case (?) todo: decide what forge should default too
forge.ScanGoSrc()
// figure out what directory we are running in
check = findPwdRepo()
@ -35,51 +35,50 @@ func main() {
log.Info("this directory isn't in a golang project (not in ~/go/src nor a go.work file)")
os.Exit(-1)
}
log.Info("starting go-mod-clean for", check.GoPath)
log.Info("go src dir is set to", forge.GetGoSrc())
if argv.Recursive {
if forge.IsGoWork() {
var warning []string
warning = append(warning, "go-mod-clean --recursive may not work unless you are in ~/go/src")
warning = append(warning, "you can continue anyway, but it hasn't been tested as much.")
simpleStdin(true, warning)
// skip restore if --force
if !argv.Force {
// try to restore from the git metadata
if restoreFromGit(check) {
okExit("go.mod was restored from the git notes")
}
var warning []string
warning = append(warning, "go-mod-clean will recreate go.mod and go.sum")
warning = append(warning, "because you have selected --recursive")
warning = append(warning, "this will redo _every_ repo. This is probably fine.")
warning = append(warning, fmt.Sprintf("You have %d total repositories in %s", forge.Repos.Len(), forge.GetGoSrc()))
warning = append(warning, "")
warning = append(warning, "However, this will also, at times, do:")
warning = append(warning, "")
warning = append(warning, "rm -rf ~/go/pkg/")
warning = append(warning, "rm -rf ~/.config/go-build/")
warning = append(warning, "")
warning = append(warning, "Which is also probably fine, but will clear all your build cache and go mod cache")
warning = append(warning, "")
simpleStdin(false, warning)
// purgeGoCaches()
} else {
simpleStdin(true, []string{"go-mod-clean will recreate go.mod and go.sum"})
}
if check.GetMasterBranchName() != check.GetCurrentBranchName() {
log.Info("")
log.Info("You can only run go-mod-clean on a git master branch.")
log.Info("Publishing go.mod & go.sum files must come from from git version tag")
log.Info("Anything else doesn't make sense.")
log.Info("")
badExit(errors.New("not git master branch"))
}
// re-create go.sum and go.mod
if _, err := check.RedoGoMod(); err != nil {
if _, err := redoGoMod(check); err != nil {
badExit(err)
}
// try to trim junk
if err := forge.TrimGoSum(check); err != nil {
badExit(err)
if argv.Trim {
// try to trim junk
if err := trimGoSum(check); err != nil {
badExit(err)
}
}
// check go.sum file
if err := forge.CleanGoDepsCheckOk(check); err != nil {
if err := cleanGoDepsCheckOk(check); err != nil {
log.Info("forge.FinalGoDepsCheck() failed. boo. :", check.GoPath)
badExit(err)
}
// put the files in the notes section in git
// this way, git commits are not messed up
// with this autogenerated code
if err := saveAsMetadata(check); err != nil {
log.Info("save go.mod as git metadata failed", check.GoPath, err)
badExit(err)
}
log.Info("forge.FinalGoDepsCheck() worked :", check.GoPath)
okExit(check.GoPath + " go.sum seems clean")
}
@ -103,12 +102,70 @@ func findPwdRepo() *gitpb.Repo {
}
func okExit(thing string) {
log.DaemonMode(true)
log.Info(thing, "ok")
log.Info("Finished go-mod-clean on", check.GetGoPath(), "ok")
// log.Info("Finished go-mod-clean on", check.GetGoPath(), "ok")
os.Exit(0)
}
func badExit(err error) {
log.DaemonMode(true)
log.Info("go-mod-clean failed: ", err, forge.GetGoSrc())
os.Exit(-1)
}
// todo: do this the right way in git
func saveAsMetadata(repo *gitpb.Repo) error {
cname := check.GetCurrentBranchName()
cmd := []string{"git", "notes", "remove", cname}
if err := check.StrictRun(cmd); err != nil {
return err
}
if check.GoPrimitive {
cmd = []string{"git", "notes", "add", "-F", "go.mod", cname}
if err := check.StrictRun(cmd); err != nil {
return err
}
} else {
cmd = []string{"git", "notes", "add", "-F", "go.mod", cname}
if err := check.StrictRun(cmd); err != nil {
return err
}
cmd = []string{"git", "notes", "append", "-m", "GOSUM:", cname}
if err := check.StrictRun(cmd); err != nil {
return err
}
cmd = []string{"git", "notes", "append", "-F", "go.sum", cname}
if err := check.StrictRun(cmd); err != nil {
return err
}
}
return nil
}
func restoreFromGit(repo *gitpb.Repo) bool {
result := repo.Run([]string{"git", "notes", "show"})
if result.Exit != 0 {
return false
}
if result.Error != nil {
return false
}
if len(result.Stdout) == 0 {
return false
}
all := strings.Join(result.Stdout, "\n")
parts := strings.Split(all, "GOSUM:")
gomod := filepath.Join(filepath.Join(check.FullPath, "go.mod"))
newf, _ := os.OpenFile(gomod, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
fmt.Fprint(newf, strings.TrimSpace(parts[0]))
if len(parts) == 2 {
gosum := filepath.Join(filepath.Join(check.FullPath, "go.sum"))
newf, _ := os.OpenFile(gosum, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
fmt.Fprint(newf, strings.TrimSpace(parts[1]))
}
return true
}

85
redoGoMod.go Normal file
View File

@ -0,0 +1,85 @@
package main
// recreates the go.mod and go.sum files
import (
"errors"
"os"
"go.wit.com/lib/protobuf/gitpb"
"go.wit.com/log"
)
// remove every go.mod and go.sum
// testing to see where this stuff is coming from
func eraseGoMod(repo *gitpb.Repo) {
// unset the go development ENV var to generate release files
if err := repo.StrictRun([]string{"rm", "-f", "go.mod", "go.sum"}); err != nil {
log.Warn(repo.GoPath, "rm go.mod go.sum failed", err)
}
}
// sets the required golang version in go.mod
func setGoVersion(repo *gitpb.Repo, version string) error {
// most things should build with golang after 1.20
if err := repo.StrictRun([]string{"go", "mod", "edit", "-go=" + version}); err != nil {
log.Warn(repo.GoPath, "go mod edit failed", err)
return err
}
return nil
}
// wrapper around 'go mod init' and 'go mod tidy'
func redoGoMod(repo *gitpb.Repo) (bool, error) {
// unset the go development ENV var to generate release files
os.Unsetenv("GO111MODULE")
if err := repo.StrictRun([]string{"rm", "-f", "go.mod", "go.sum"}); err != nil {
log.Warn("rm go.mod go.sum failed", err)
return false, err
}
if err := repo.StrictRun([]string{"go", "mod", "init", repo.GoPath}); err != nil {
log.Warn("go mod init failed", err)
return false, err
}
if err := repo.StrictRun([]string{"go", "mod", "tidy"}); err != nil {
log.Warn("go mod tidy failed", err)
return false, err
}
// most things should build with golang after 1.20 // todo: allow this to be set somewhere
if err := setGoVersion(repo, "1.20"); err != nil {
log.Warn(repo.GoPath, "go mod edit failed", err)
return false, err
}
repo.GoDeps = nil
repo.GoPrimitive = false
// if there is not a go.sum file, it better be a primitive golang project
if !repo.Exists("go.sum") {
// this should never happen
return false, errors.New("MakeRedomod() logic failed")
ok, err := repo.IsPrimitive()
if err != nil {
// this means this repo does not depend on any other package
log.Info("PRIMATIVE repo error:", repo.GoPath, "err =", err)
return false, err
}
if ok {
// this means the repo is primitive so there is no go.sum
repo.GoPrimitive = true
return true, nil
}
}
repo.GoDeps = new(gitpb.GoDeps)
if !repo.Exists("go.sum") {
// this should never happen
return false, errors.New("MakeRedomod() logic failed")
}
// return the attempt to parse go.sum
return repo.ParseGoSum()
}