From 4582a76081814fa8bc573ad003c132d41347d6f4 Mon Sep 17 00:00:00 2001 From: Jeff Carr Date: Wed, 18 Dec 2024 17:59:20 -0600 Subject: [PATCH] finally re-organize and rename Clone to GoClone() --- clone.go | 278 +++++++++++++++++++++++---------------------------- goSrcFind.go | 6 +- repoNew.go | 22 ++++ run.go | 54 ++++++++++ 4 files changed, 202 insertions(+), 158 deletions(-) create mode 100644 run.go diff --git a/clone.go b/clone.go index 43ee482..8225c58 100644 --- a/clone.go +++ b/clone.go @@ -2,193 +2,171 @@ package forgepb import ( "errors" + "fmt" "io/ioutil" "net/http" - "os" "path/filepath" "strings" - "go.wit.com/lib/gui/shell" "go.wit.com/lib/protobuf/gitpb" "go.wit.com/log" ) -// TODO: make some config file for things like this -// can be used to work around temporary problems -func clonePathHack(dirname string, basedir string, gopath string) (string, error) { - // newdir = helloworld - // basedir = /home/jcarr/go/src/go.wit.com/apps - // giturl = https://gitea.wit.com/gui/helloworld - // func cloneActual(newdir, basedir, giturl string) error { +// will not violate filesystem namespace +// always returns the path or a parent path +// +// attemps to exec git clone based off of a golang path +// will transferse parent directories in case the path +// is a child of a git repo +// +// returns *gitpb.Repo if already cloned +// +// example gopath = go.wit.com/apps/go-clone +// or "go.googlesource.com/go/src/cmd/internal/pkgpath/" returns repo for "go.googlesource.com/go" +func (f *Forge) GoClone(gopath string) (*gitpb.Repo, error) { - switch gopath { - case "golang.org/x/crypto": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/crypto") - case "golang.org/x/mod": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/mod") - case "golang.org/x/net": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/net") - case "golang.org/x/sys": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sys") - case "golang.org/x/sync": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sync") - case "golang.org/x/term": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/term") - case "golang.org/x/text": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/text") - case "golang.org/x/tools": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/tools") - case "golang.org/x/xerrors": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/xerrors") - case "google.golang.org/protobuf": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/protobuf") - case "google.golang.org/genproto": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/genproto") - case "google.golang.org/api": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/api") - case "google.golang.org/grpc": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/grpc") - case "google.golang.org/appengine": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/appengine") - } - if strings.HasPrefix(gopath, "github.com/go-gl/glfw") { - return cloneActual(dirname, basedir, "https://github.com/go-gl/glfw") - } - - return "", errors.New("no gopath override here") -} - -// attempt to git clone if the go path doesn't exist -// does a git clone, if it works, returns true -// gopath = go.wit.com/apps/helloworld -func (f *Forge) Clone(gopath string) (*gitpb.Repo, error) { - var err error + // will match /root/go/src/go.wit.com/apps/go-clone/something/inside + // and return the *gitpb.Repo for "go.wit.com/apps/go-clone" fullpath := filepath.Join(f.goSrc, gopath) - - if pb := f.Repos.FindByFullPath(fullpath); pb != nil { + if pb := f.FindAnyPath(fullpath); pb != nil { // repo already exists return pb, nil } - dirname := filepath.Base(fullpath) - basedir := strings.TrimSuffix(fullpath, dirname) - - url := "https://" + gopath - log.Info("trying git clone") - log.Info("gopath =", gopath) - // try a direct git clone against the gopath - // cloneActual("helloworld", "/home/jcarr/go/src/go.wit.com/apps", "https://go.wit.com/apps/helloworld") - if finalurl, err := cloneActual(dirname, basedir, url); err == nil { - return f.finishClone(gopath, finalurl) + // if this doesn't work, probably the package is abandoned and you + // are probably using something old, broken, or wrong + if repo, err := f.urlClone(gopath, "https://"+gopath); repo != nil { + return repo, err } - log.Info("direct attempt at git clone failed", url) - // if direct git clone doesn't work, look for a redirect - // go directly to the URL as that is autoritive. If that failes - // try the go package system as maybe the git site no longer exists - if url, err = findGoImport(url); err != nil { - log.Info("findGoImport() DID NOT WORK", url) - log.Info("findGoImport() DID NOT WORK", err) - } else { - if finalurl, err := cloneActual(dirname, basedir, url); err == nil { - return f.finishClone(gopath, finalurl) - } + // check for parent git repos + if repo, err := f.goClonePop(gopath); repo != nil { + return repo, err } - log.Info("git clone from 'go-import' info failed", url) // query the golang package system for the last known location // NOTE: take time to thank the go developers and google for designing this wonderful system - if url, err = runGoList(gopath); err != nil { - log.Info("go list failed", err) - } else { - if finalurl, err := cloneActual(dirname, basedir, url); err == nil { - return f.finishClone(gopath, finalurl) + if pkgurl, err := runGoList(gopath); err == nil { + if repo, err := f.urlClone(gopath, pkgurl); repo != nil { + return repo, err } } - log.Info("git clone from 'git list' info failed", url) - // try to parse a redirect + // todo: emit some sort of warning? - if finalurl, err := clonePathHack(dirname, basedir, gopath); err == nil { - return f.finishClone(gopath, finalurl) + // hacks + if repo, err := f.clonePathHack(gopath); repo != nil { + return repo, err } return nil, errors.New("can not find git sources for gopath " + gopath) } -// actually does something smart -func (f *Forge) finishClone(gopath string, giturl string) (*gitpb.Repo, error) { - var err error - newr := f.FindByGoPath(gopath) - if newr == nil { - newr, err = f.NewGoRepo(gopath, giturl) +// this is obvious a experiemental hack +// todo: make a config file for this? +func overridePath(gopath string) string { + switch gopath { + case "golang.org/x/crypto": + return "https://" + "go.googlesource.com/crypto" + case "golang.org/x/mod": + return "https://" + "go.googlesource.com/mod" + case "golang.org/x/net": + return "https://" + "go.googlesource.com/net" + case "golang.org/x/sys": + return "https://" + "go.googlesource.com/sys" + case "golang.org/x/sync": + return "https://" + "go.googlesource.com/sync" + case "golang.org/x/term": + return "https://" + "go.googlesource.com/term" + case "golang.org/x/text": + return "https://" + "go.googlesource.com/text" + case "golang.org/x/tools": + return "https://" + "go.googlesource.com/tools" + case "golang.org/x/xerrors": + return "https://" + "go.googlesource.com/xerrors" + case "google.golang.org/protobuf": + return "https://" + "go.googlesource.com/protobuf" + case "google.golang.org/genproto": + return "https://" + "go.googlesource.com/genproto" + case "google.golang.org/api": + return "https://" + "go.googlesource.com/api" + case "google.golang.org/grpc": + return "https://" + "go.googlesource.com/grpc" + case "google.golang.org/appengine": + return "https://" + "go.googlesource.com/appengine" } - if newr == nil { - log.Warn("forge.Clone() new repo can not be found or created for gopath", gopath) - return nil, err + if strings.HasPrefix(gopath, "github.com/go-gl/glfw") { + return "https://github.com/go-gl/glfw" } - if newr.URL != giturl { - log.Warn("forge.Clone() url changed", newr.URL, "to", giturl) - newr.URL = giturl - } - if err := newr.RepoIgnoresGoMod(); err != nil { - log.Info("never modify go.mod or go.sum for this repo", newr.GetGoPath()) - log.Info("We recommend you add 'go.*' to your .gitignore file and store those files as git tag metadata") - newr.ParseGoSum() - return newr, nil - } - - if newr.Exists("go.mod") { - return newr, nil - } - - log.Info("todo: something went wrong probably. didn't finish. run go-mod-clean? (can't here. loop of circles)") - log.Info("todo: do go mod init here directly") - log.Info("todo: try to run go mod init here", newr.GetGoPath()) - return newr, nil + return "" } +// TODO: make some config file for things like this +// can be used to work around temporary problems +func (f *Forge) clonePathHack(gopath string) (*gitpb.Repo, error) { + // newdir = helloworld + // basedir = /home/jcarr/go/src/go.wit.com/apps + // giturl = https://gitea.wit.com/gui/helloworld + + url := overridePath(gopath) + if url == "" { + return nil, errors.New("no gopath override here") + } + + return f.urlClone(gopath, url) +} + +// for: github.com/gdamore/tcell/v2 +// tries git clone github.com/gdamore/tcell/v2 +// then git clone github.com/gdamore/tcell +// then git clone github.com/gdamore , etc +func (f *Forge) goClonePop(gopath string) (*gitpb.Repo, error) { + log.Info("forge.goClonePop() trying", gopath) + if gopath == "" { + return nil, nil + } + fullpath := filepath.Join(f.GetGoSrc(), gopath) + if pb := f.FindAnyPath(fullpath); pb != nil { + // repo already exists + return pb, nil + } + + newpath, _ := filepath.Split(gopath) + if repo, _ := f.urlClone(newpath, "https://"+newpath); repo != nil { + return repo, nil + } + + if repo, err := f.goClonePop(newpath); repo != nil { + return repo, err + } + return nil, fmt.Errorf("forge.goClonePop() failed %s", gopath) +} + +// clone a URL directly, also try cloning if 'go-import' is sent // newdir = helloworld // basedir = /home/jcarr/go/src/go.wit.com/apps // giturl = https://gitea.wit.com/gui/helloworld -func cloneActual(newdir, basedir, giturl string) (string, error) { - log.Info("cloneActual() newdir =", newdir) - log.Info("cloneActual() basedir =", basedir) - log.Info("cloneActual() giturl =", giturl) - if !IsDirectory(basedir) { - os.MkdirAll(basedir, 0750) - } - err := os.Chdir(basedir) - if err != nil { - log.Warn("chdir failed", basedir, err) - return giturl, err +func (f *Forge) urlClone(gopath, giturl string) (*gitpb.Repo, error) { + var err error + + fullpath := filepath.Join(f.goSrc, gopath) + basedir, newdir := filepath.Split(fullpath) + + // clone the URL directly + if err = RunGitClone(newdir, basedir, giturl); err == nil { + return f.NewGoRepo(gopath, giturl) } - cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir} - log.Info("Running:", basedir, cmd) - r := shell.PathRunRealtime(basedir, cmd) - if r.Error != nil { - log.Warn("git clone error", r.Error) - return giturl, r.Error + // see if the URL has go-import for a new URL + if giturl, err = findGoImport(giturl); err == nil { + if err = RunGitClone(newdir, basedir, giturl); err == nil { + return f.NewGoRepo(gopath, giturl) + } } - - fullpath := filepath.Join(basedir, newdir) - if !IsDirectory(fullpath) { - log.Info("git clone failed", giturl) - return giturl, errors.New("git clone failed " + giturl) - } - gitdir := filepath.Join(fullpath, ".git") - if IsDirectory(gitdir) { - log.Info("git cloned worked to", fullpath) - // also clone notes -- this can store the go.mod and go.sum files - cmd := []string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"} - shell.PathRunRealtime(fullpath, cmd) - return giturl, nil - } - // git clone didn't really work but did make a directory - log.Info("fullpath is probably empty", fullpath) - return giturl, errors.New("crapnuts. rmdir fullpath here? " + fullpath) + // log.Info("git clone from 'go-import' info failed", url) + // yes, this misses the first error + return nil, err } // check the server for the current go path to git url mapping @@ -236,11 +214,3 @@ func findGoImport(url string) (string, error) { return newurl, nil } - -func IsDirectory(path string) bool { - info, err := os.Stat(path) - if err != nil { - return false - } - return info.IsDir() -} diff --git a/goSrcFind.go b/goSrcFind.go index 7dbb26f..b74c7d9 100644 --- a/goSrcFind.go +++ b/goSrcFind.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" - "go.wit.com/lib/gui/shell" "go.wit.com/log" ) @@ -48,9 +47,8 @@ func useGoSrc() (string, error) { return "", err } pwd := filepath.Join(homeDir, "go/src") - shell.Mkdir(pwd) - // os.Chdir(pwd) - return pwd, nil + err = os.MkdirAll(pwd, os.ModePerm) + return pwd, err } func (f *Forge) goWorkExists() bool { diff --git a/repoNew.go b/repoNew.go index 45e9db5..a089ab7 100644 --- a/repoNew.go +++ b/repoNew.go @@ -143,6 +143,28 @@ func (f *Forge) FindByGoPath(gopath string) *gitpb.Repo { return f.Repos.FindByFullPath(fullpath) } +// tries any parent +func (f *Forge) FindAnyPath(dir string) *gitpb.Repo { + dir = filepath.Clean(dir) + repo := f.Repos.FindByFullPath(dir) + if repo != nil { + log.Info("FindAnyPath() worked = ", dir) + return repo + } + if dir == "" { + return nil + } + if dir == "/" { + return nil + } + basedir, newdir := filepath.Split(dir) + if newdir == "" { + // is this correct? not sure what stupid windows does + return nil + } + return f.FindAnyPath(basedir) +} + func (f *Forge) DeleteByGoPath(gopath string) bool { fullpath := filepath.Join(f.GetGoSrc(), gopath) return f.Repos.DeleteByFullPath(fullpath) diff --git a/run.go b/run.go new file mode 100644 index 0000000..2f2e57f --- /dev/null +++ b/run.go @@ -0,0 +1,54 @@ +package forgepb + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "go.wit.com/lib/gui/shell" + "go.wit.com/log" +) + +// git clone (also downloads git notes) +// newdir = helloworld +// basedir = /home/jcarr/go/src/go.wit.com/apps +// giturl = https://gitea.wit.com/gui/helloworld +func RunGitClone(newdir, basedir, giturl string) error { + log.Info("runGitClone() newdir =", newdir) + log.Info("runGitClone() basedir =", basedir) + log.Info("runGitClone() giturl =", giturl) + if !shell.IsDir(basedir) { + os.MkdirAll(basedir, os.ModePerm) + } + err := os.Chdir(basedir) + if err != nil { + // log.Warn("chdir failed", basedir, err) + return err + } + + cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir} + log.Info("Running:", basedir, cmd) + r := shell.PathRunRealtime(basedir, cmd) + if r.Error != nil { + // log.Warn("git clone error", r.Error) + return r.Error + } + + fullpath := filepath.Join(basedir, newdir) + if !shell.IsDir(fullpath) { + // log.Info("git clone failed", giturl) + return fmt.Errorf("git clone %s failed to create a directory", giturl) + } + gitdir := filepath.Join(fullpath, ".git") + if shell.IsDir(gitdir) { + // log.Info("git cloned worked to", fullpath) + // also clone notes -- this can store the go.mod and go.sum files + cmd := []string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"} + shell.PathRunRealtime(fullpath, cmd) + return nil + } + // git clone didn't really work but did make a directory + log.Info("fullpath is probably empty", fullpath) + return errors.New("crapnuts. rmdir fullpath here? " + fullpath) +}