finally re-organize and rename Clone to GoClone()

clone.go

@@ -2,193 +2,171 @@ package forgepb
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strings"
 
-	"go.wit.com/lib/gui/shell"
 	"go.wit.com/lib/protobuf/gitpb"
 	"go.wit.com/log"
 )
 
-// TODO: make some config file for things like this
+// will not violate filesystem namespace
-// can be used to work around temporary problems
+// always returns the path or a parent path
-func clonePathHack(dirname string, basedir string, gopath string) (string, error) {
+//
-	// newdir = helloworld
+// attemps to exec git clone based off of a golang path
-	// basedir = /home/jcarr/go/src/go.wit.com/apps
+// will transferse parent directories in case the path
-	// giturl = https://gitea.wit.com/gui/helloworld
+// is a child of a git repo
-	// func cloneActual(newdir, basedir, giturl string) error {
+//
+// returns *gitpb.Repo if already cloned
+//
+// example gopath  = go.wit.com/apps/go-clone
+// or "go.googlesource.com/go/src/cmd/internal/pkgpath/" returns repo for "go.googlesource.com/go"
+func (f *Forge) GoClone(gopath string) (*gitpb.Repo, error) {
 
-	switch gopath {
+	// will match /root/go/src/go.wit.com/apps/go-clone/something/inside
-	case "golang.org/x/crypto":
+	// and return the *gitpb.Repo for "go.wit.com/apps/go-clone"
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/crypto")
-	case "golang.org/x/mod":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/mod")
-	case "golang.org/x/net":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/net")
-	case "golang.org/x/sys":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sys")
-	case "golang.org/x/sync":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sync")
-	case "golang.org/x/term":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/term")
-	case "golang.org/x/text":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/text")
-	case "golang.org/x/tools":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/tools")
-	case "golang.org/x/xerrors":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/xerrors")
-	case "google.golang.org/protobuf":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/protobuf")
-	case "google.golang.org/genproto":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/genproto")
-	case "google.golang.org/api":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/api")
-	case "google.golang.org/grpc":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/grpc")
-	case "google.golang.org/appengine":
-		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/appengine")
-	}
-	if strings.HasPrefix(gopath, "github.com/go-gl/glfw") {
-		return cloneActual(dirname, basedir, "https://github.com/go-gl/glfw")
-	}
-
-	return "", errors.New("no gopath override here")
-}
-
-// attempt to git clone if the go path doesn't exist
-// does a git clone, if it works, returns true
-// gopath  = go.wit.com/apps/helloworld
-func (f *Forge) Clone(gopath string) (*gitpb.Repo, error) {
-	var err error
 	fullpath := filepath.Join(f.goSrc, gopath)
-
+	if pb := f.FindAnyPath(fullpath); pb != nil {
-	if pb := f.Repos.FindByFullPath(fullpath); pb != nil {
 		// repo already exists
 		return pb, nil
 	}
 
-	dirname := filepath.Base(fullpath)
-	basedir := strings.TrimSuffix(fullpath, dirname)
-
-	url := "https://" + gopath
-	log.Info("trying git clone")
-	log.Info("gopath =", gopath)
-
 	// try a direct git clone against the gopath
-	// cloneActual("helloworld", "/home/jcarr/go/src/go.wit.com/apps", "https://go.wit.com/apps/helloworld")
+	// if this doesn't work, probably the package is abandoned and you
-	if finalurl, err := cloneActual(dirname, basedir, url); err == nil {
+	// are probably using something old, broken, or wrong
-		return f.finishClone(gopath, finalurl)
+	if repo, err := f.urlClone(gopath, "https://"+gopath); repo != nil {
+		return repo, err
 	}
-	log.Info("direct attempt at git clone failed", url)
 
-	// if direct git clone doesn't work, look for a redirect
+	// check for parent git repos
-	// go directly to the URL as that is autoritive. If that failes
+	if repo, err := f.goClonePop(gopath); repo != nil {
-	// try the go package system as maybe the git site no longer exists
+		return repo, err
-	if url, err = findGoImport(url); err != nil {
-		log.Info("findGoImport() DID NOT WORK", url)
-		log.Info("findGoImport() DID NOT WORK", err)
-	} else {
-		if finalurl, err := cloneActual(dirname, basedir, url); err == nil {
-			return f.finishClone(gopath, finalurl)
 	}
-	}
-	log.Info("git clone from 'go-import' info failed", url)
 
 	// query the golang package system for the last known location
 	// NOTE: take time to thank the go developers and google for designing this wonderful system
-	if url, err = runGoList(gopath); err != nil {
+	if pkgurl, err := runGoList(gopath); err == nil {
-		log.Info("go list failed", err)
+		if repo, err := f.urlClone(gopath, pkgurl); repo != nil {
-	} else {
+			return repo, err
-		if finalurl, err := cloneActual(dirname, basedir, url); err == nil {
-			return f.finishClone(gopath, finalurl)
 		}
 	}
-	log.Info("git clone from 'git list' info failed", url)
 
-	// try to parse a redirect
+	// todo: emit some sort of warning?
 
-	if finalurl, err := clonePathHack(dirname, basedir, gopath); err == nil {
+	// hacks
-		return f.finishClone(gopath, finalurl)
+	if repo, err := f.clonePathHack(gopath); repo != nil {
+		return repo, err
 	}
 
 	return nil, errors.New("can not find git sources for gopath " + gopath)
 }
 
-// actually does something smart
+// this is obvious a experiemental hack
-func (f *Forge) finishClone(gopath string, giturl string) (*gitpb.Repo, error) {
+// todo: make a config file for this?
-	var err error
+func overridePath(gopath string) string {
-	newr := f.FindByGoPath(gopath)
+	switch gopath {
-	if newr == nil {
+	case "golang.org/x/crypto":
-		newr, err = f.NewGoRepo(gopath, giturl)
+		return "https://" + "go.googlesource.com/crypto"
+	case "golang.org/x/mod":
+		return "https://" + "go.googlesource.com/mod"
+	case "golang.org/x/net":
+		return "https://" + "go.googlesource.com/net"
+	case "golang.org/x/sys":
+		return "https://" + "go.googlesource.com/sys"
+	case "golang.org/x/sync":
+		return "https://" + "go.googlesource.com/sync"
+	case "golang.org/x/term":
+		return "https://" + "go.googlesource.com/term"
+	case "golang.org/x/text":
+		return "https://" + "go.googlesource.com/text"
+	case "golang.org/x/tools":
+		return "https://" + "go.googlesource.com/tools"
+	case "golang.org/x/xerrors":
+		return "https://" + "go.googlesource.com/xerrors"
+	case "google.golang.org/protobuf":
+		return "https://" + "go.googlesource.com/protobuf"
+	case "google.golang.org/genproto":
+		return "https://" + "go.googlesource.com/genproto"
+	case "google.golang.org/api":
+		return "https://" + "go.googlesource.com/api"
+	case "google.golang.org/grpc":
+		return "https://" + "go.googlesource.com/grpc"
+	case "google.golang.org/appengine":
+		return "https://" + "go.googlesource.com/appengine"
 	}
-	if newr == nil {
+	if strings.HasPrefix(gopath, "github.com/go-gl/glfw") {
-		log.Warn("forge.Clone() new repo can not be found or created for gopath", gopath)
+		return "https://github.com/go-gl/glfw"
-		return nil, err
 	}
-	if newr.URL != giturl {
+	return ""
-		log.Warn("forge.Clone() url changed", newr.URL, "to", giturl)
-		newr.URL = giturl
-	}
-	if err := newr.RepoIgnoresGoMod(); err != nil {
-		log.Info("never modify go.mod or go.sum for this repo", newr.GetGoPath())
-		log.Info("We recommend you add 'go.*' to your .gitignore file and store those files as git tag metadata")
-		newr.ParseGoSum()
-		return newr, nil
-	}
-
-	if newr.Exists("go.mod") {
-		return newr, nil
-	}
-
-	log.Info("todo: something went wrong probably. didn't finish. run go-mod-clean? (can't here. loop of circles)")
-	log.Info("todo: do go mod init here directly")
-	log.Info("todo: try to run go mod init here", newr.GetGoPath())
-	return newr, nil
 }
 
+// TODO: make some config file for things like this
+// can be used to work around temporary problems
+func (f *Forge) clonePathHack(gopath string) (*gitpb.Repo, error) {
 	// newdir = helloworld
 	// basedir = /home/jcarr/go/src/go.wit.com/apps
 	// giturl = https://gitea.wit.com/gui/helloworld
-func cloneActual(newdir, basedir, giturl string) (string, error) {
+
-	log.Info("cloneActual() newdir  =", newdir)
+	url := overridePath(gopath)
-	log.Info("cloneActual() basedir =", basedir)
+	if url == "" {
-	log.Info("cloneActual() giturl  =", giturl)
+		return nil, errors.New("no gopath override here")
-	if !IsDirectory(basedir) {
-		os.MkdirAll(basedir, 0750)
-	}
-	err := os.Chdir(basedir)
-	if err != nil {
-		log.Warn("chdir failed", basedir, err)
-		return giturl, err
 	}
 
-	cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir}
+	return f.urlClone(gopath, url)
-	log.Info("Running:", basedir, cmd)
-	r := shell.PathRunRealtime(basedir, cmd)
-	if r.Error != nil {
-		log.Warn("git clone error", r.Error)
-		return giturl, r.Error
 }
 
-	fullpath := filepath.Join(basedir, newdir)
+// for: github.com/gdamore/tcell/v2
-	if !IsDirectory(fullpath) {
+// tries git clone github.com/gdamore/tcell/v2
-		log.Info("git clone failed", giturl)
+// then  git clone github.com/gdamore/tcell
-		return giturl, errors.New("git clone failed " + giturl)
+// then  git clone github.com/gdamore , etc
+func (f *Forge) goClonePop(gopath string) (*gitpb.Repo, error) {
+	log.Info("forge.goClonePop() trying", gopath)
+	if gopath == "" {
+		return nil, nil
 	}
-	gitdir := filepath.Join(fullpath, ".git")
+	fullpath := filepath.Join(f.GetGoSrc(), gopath)
-	if IsDirectory(gitdir) {
+	if pb := f.FindAnyPath(fullpath); pb != nil {
-		log.Info("git cloned worked to", fullpath)
+		// repo already exists
-		// also clone notes -- this can store the go.mod and go.sum files
+		return pb, nil
-		cmd := []string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"}
-		shell.PathRunRealtime(fullpath, cmd)
-		return giturl, nil
 	}
-	// git clone didn't really work but did make a directory
+
-	log.Info("fullpath is probably empty", fullpath)
+	newpath, _ := filepath.Split(gopath)
-	return giturl, errors.New("crapnuts. rmdir fullpath here? " + fullpath)
+	if repo, _ := f.urlClone(newpath, "https://"+newpath); repo != nil {
+		return repo, nil
+	}
+
+	if repo, err := f.goClonePop(newpath); repo != nil {
+		return repo, err
+	}
+	return nil, fmt.Errorf("forge.goClonePop() failed %s", gopath)
+}
+
+// clone a URL directly, also try cloning if 'go-import' is sent
+// newdir = helloworld
+// basedir = /home/jcarr/go/src/go.wit.com/apps
+// giturl = https://gitea.wit.com/gui/helloworld
+func (f *Forge) urlClone(gopath, giturl string) (*gitpb.Repo, error) {
+	var err error
+
+	fullpath := filepath.Join(f.goSrc, gopath)
+	basedir, newdir := filepath.Split(fullpath)
+
+	// clone the URL directly
+	if err = RunGitClone(newdir, basedir, giturl); err == nil {
+		return f.NewGoRepo(gopath, giturl)
+	}
+
+	// see if the URL has go-import for a new URL
+	if giturl, err = findGoImport(giturl); err == nil {
+		if err = RunGitClone(newdir, basedir, giturl); err == nil {
+			return f.NewGoRepo(gopath, giturl)
+		}
+	}
+	// log.Info("git clone from 'go-import' info failed", url)
+	// yes, this misses the first error
+	return nil, err
 }
 
 // check the server for the current go path to git url mapping
@@ -236,11 +214,3 @@ func findGoImport(url string) (string, error) {
 
 	return newurl, nil
 }
-
-func IsDirectory(path string) bool {
-	info, err := os.Stat(path)
-	if err != nil {
-		return false
-	}
-	return info.IsDir()
-}

goSrcFind.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"go.wit.com/lib/gui/shell"
 	"go.wit.com/log"
 )
 
@@ -48,9 +47,8 @@ func useGoSrc() (string, error) {
 		return "", err
 	}
 	pwd := filepath.Join(homeDir, "go/src")
-	shell.Mkdir(pwd)
+	err = os.MkdirAll(pwd, os.ModePerm)
-	// os.Chdir(pwd)
+	return pwd, err
-	return pwd, nil
 }
 
 func (f *Forge) goWorkExists() bool {

repoNew.go

@@ -143,6 +143,28 @@ func (f *Forge) FindByGoPath(gopath string) *gitpb.Repo {
 	return f.Repos.FindByFullPath(fullpath)
 }
 
+// tries any parent
+func (f *Forge) FindAnyPath(dir string) *gitpb.Repo {
+	dir = filepath.Clean(dir)
+	repo := f.Repos.FindByFullPath(dir)
+	if repo != nil {
+		log.Info("FindAnyPath() worked = ", dir)
+		return repo
+	}
+	if dir == "" {
+		return nil
+	}
+	if dir == "/" {
+		return nil
+	}
+	basedir, newdir := filepath.Split(dir)
+	if newdir == "" {
+		// is this correct? not sure what stupid windows does
+		return nil
+	}
+	return f.FindAnyPath(basedir)
+}
+
 func (f *Forge) DeleteByGoPath(gopath string) bool {
 	fullpath := filepath.Join(f.GetGoSrc(), gopath)
 	return f.Repos.DeleteByFullPath(fullpath)

run.go

@@ -0,0 +1,54 @@
+package forgepb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"go.wit.com/lib/gui/shell"
+	"go.wit.com/log"
+)
+
+// git clone (also downloads git notes)
+// newdir = helloworld
+// basedir = /home/jcarr/go/src/go.wit.com/apps
+// giturl = https://gitea.wit.com/gui/helloworld
+func RunGitClone(newdir, basedir, giturl string) error {
+	log.Info("runGitClone() newdir  =", newdir)
+	log.Info("runGitClone() basedir =", basedir)
+	log.Info("runGitClone() giturl  =", giturl)
+	if !shell.IsDir(basedir) {
+		os.MkdirAll(basedir, os.ModePerm)
+	}
+	err := os.Chdir(basedir)
+	if err != nil {
+		// log.Warn("chdir failed", basedir, err)
+		return err
+	}
+
+	cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir}
+	log.Info("Running:", basedir, cmd)
+	r := shell.PathRunRealtime(basedir, cmd)
+	if r.Error != nil {
+		// log.Warn("git clone error", r.Error)
+		return r.Error
+	}
+
+	fullpath := filepath.Join(basedir, newdir)
+	if !shell.IsDir(fullpath) {
+		// log.Info("git clone failed", giturl)
+		return fmt.Errorf("git clone %s failed to create a directory", giturl)
+	}
+	gitdir := filepath.Join(fullpath, ".git")
+	if shell.IsDir(gitdir) {
+		// log.Info("git cloned worked to", fullpath)
+		// also clone notes -- this can store the go.mod and go.sum files
+		cmd := []string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"}
+		shell.PathRunRealtime(fullpath, cmd)
+		return nil
+	}
+	// git clone didn't really work but did make a directory
+	log.Info("fullpath is probably empty", fullpath)
+	return errors.New("crapnuts. rmdir fullpath here? " + fullpath)
+}