move Clone() here

config.go

@@ -111,7 +111,7 @@ func (c *ForgeConfigs) ConfigLoad() error {
 	}
 
 	// first time user. make a template config file
-	c.SampleConfig()
+	c.sampleConfig()
 
 	return nil
 }

configDefault.go

@@ -6,7 +6,7 @@ import (
 	"go.wit.com/log"
 )
 
-func (all *ForgeConfigs) SampleConfig() {
+func (all *ForgeConfigs) sampleConfig() {
 	new1 := new(ForgeConfig)
 	new1.GoPath = "go.wit.com"
 	new1.Writable = true

forgeConfig/main.go

@@ -15,7 +15,7 @@ func main() {
 
 	if argv.List {
 		f.ConfigPrintTable()
-		loop := f.SortByPath() // get the list of forge configs
+		loop := f.Config.SortByPath() // get the list of forge configs
 		for loop.Scan() {
 			r := loop.Next()
 			log.Info("repo:", r.GoPath)

goList.go

@@ -0,0 +1,117 @@
+package forgepb
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go.wit.com/lib/gui/shell"
+	"go.wit.com/log"
+)
+
+// go list -json -m go.wit.com/apps/go-clone@latest
+
+// go list -json -m go.wit.com/apps/go-clone@latest
+// {
+// 	"Path": "go.wit.com/apps/go-clone",
+// 	"Version": "v0.0.6",
+// 	"Query": "latest",
+// 	"Time": "2024-03-10T04:12:15Z",
+// 	"GoMod": "/home/jcarr/go/pkg/mod/cache/download/go.wit.com/apps/go-clone/@v/v0.0.6.mod",
+// 	"GoVersion": "1.22.0"
+// }
+
+type Module struct {
+	Path       string       // module path
+	Query      string       // version query corresponding to this version
+	Version    string       // module version
+	Versions   []string     // available module versions
+	Replace    *Module      // replaced by this module
+	Time       *time.Time   // time version was created
+	Update     *Module      // available update (with -u)
+	Main       bool         // is this the main module?
+	Indirect   bool         // module is only indirectly needed by main module
+	Dir        string       // directory holding local copy of files, if any
+	GoMod      string       // path to go.mod file describing module, if any
+	GoVersion  string       // go version used in module
+	Retracted  []string     // retraction information, if any (with -retracted or -u)
+	Deprecated string       // deprecation message, if any (with -u)
+	Error      *ModuleError // error loading module
+	Sum        string       // checksum for path, version (as in go.sum)
+	GoModSum   string       // checksum for go.mod (as in go.sum)
+	Reuse      bool         // reuse of old module info is safe
+	Origin     Origin
+}
+
+type ModuleError struct {
+	Err string // the error itself
+}
+
+// An Origin describes the provenance of a given repo method result.
+// It can be passed to CheckReuse (usually in a different go command invocation)
+// to see whether the result remains up-to-date.
+type Origin struct {
+	VCS    string `json:",omitempty"` // "git" etc
+	URL    string `json:",omitempty"` // URL of repository
+	Subdir string `json:",omitempty"` // subdirectory in repo
+
+	Hash string `json:",omitempty"` // commit hash or ID
+
+	// If TagSum is non-empty, then the resolution of this module version
+	// depends on the set of tags present in the repo, specifically the tags
+	// of the form TagPrefix + a valid semver version.
+	// If the matching repo tags and their commit hashes still hash to TagSum,
+	// the Origin is still valid (at least as far as the tags are concerned).
+	// The exact checksum is up to the Repo implementation; see (*gitRepo).Tags.
+	TagPrefix string `json:",omitempty"`
+	TagSum    string `json:",omitempty"`
+
+	// If Ref is non-empty, then the resolution of this module version
+	// depends on Ref resolving to the revision identified by Hash.
+	// If Ref still resolves to Hash, the Origin is still valid (at least as far as Ref is concerned).
+	// For Git, the Ref is a full ref like "refs/heads/main" or "refs/tags/v1.2.3",
+	// and the Hash is the Git object hash the ref maps to.
+	// Other VCS might choose differently, but the idea is that Ref is the name
+	// with a mutable meaning while Hash is a name with an immutable meaning.
+	Ref string `json:",omitempty"`
+
+	// If RepoSum is non-empty, then the resolution of this module version
+	// failed due to the repo being available but the version not being present.
+	// This depends on the entire state of the repo, which RepoSum summarizes.
+	// For Git, this is a hash of all the refs and their hashes.
+	RepoSum string `json:",omitempty"`
+}
+
+// A Tags describes the available tags in a code repository.
+
+func runGoList(url string) (string, error) {
+	ver, err := getLatestVersion(url)
+	if err != nil {
+		return "", err
+	}
+	r := shell.Run([]string{"go", "list", "-json", "-m", url + "@" + ver})
+	var modInfo Module
+	out := strings.Join(r.Stdout, "\n")
+	err = json.Unmarshal([]byte(out), &modInfo)
+	if err != nil {
+		log.Info("runGoList() r.Output =", out)
+		log.Info("runGoList() json.Unmarshal() error =", err)
+		return "", err
+	}
+	log.Info("runGoList() Output.Origin.URL =", modInfo.Origin.URL)
+	return modInfo.Origin.URL, err
+}
+
+func getLatestVersion(url string) (string, error) {
+	r := shell.Run([]string{"go", "list", "-json", "-m", url + "@latest"})
+	var modInfo Module
+	out := strings.Join(r.Stdout, "\n")
+	err := json.Unmarshal([]byte(out), &modInfo)
+	if err != nil {
+		log.Info("runGoList() r.Output =", out)
+		log.Info("runGoList() json.Unmarshal() error =", err)
+		return "", err
+	}
+	log.Info("runGoList() Output.Version =", modInfo.Version)
+	return modInfo.Version, err
+}

init.go

@@ -50,7 +50,3 @@ func Init() *Forge {
 	log.Warn("GOT HERE. forge.Init(). f can not be nil")
 	return f
 }
-
-func (f *Forge) SortByPath() *ForgeConfigIterator {
-	return f.Config.SortByPath()
-}

repoClone.go

@@ -0,0 +1,278 @@
+package forgepb
+
+import (
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go.wit.com/lib/gui/shell"
+	"go.wit.com/lib/protobuf/gitpb"
+	"go.wit.com/log"
+)
+
+/*
+// guess the paths. returns
+
+// realpath : the actual path on the filesystem
+// goSrcPath : this could be ~/go/src, or where the go.work file is
+
+// goPath : go.wit.com/lib/gui/repostatus (for example)
+// true/false if the repo is a golang repo
+func (f *Forge) guessPaths(path string) (string, string, string, bool, error) {
+	var realpath, goSrcDir string
+	var isGoLang bool = false
+
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		log.Log(FORGEPBWARN, "Error getting home directory:", err)
+		return path, realpath, goSrcDir, false, err
+	}
+	goSrcDir = filepath.Join(homeDir, "go/src")
+
+	// allow arbitrary paths, otherwise, assume the repo is in ~/go/src
+	// unless REPO_WORK_PATH was set. to over-ride ~/go/src
+	// todo, look for go.work
+	if os.Getenv("REPO_WORK_PATH") == "" {
+		os.Setenv("REPO_WORK_PATH", goSrcDir)
+	} else {
+		goSrcDir = os.Getenv("REPO_WORK_PATH")
+	}
+
+	// todo: this is dumb
+	if strings.HasPrefix(path, "/") {
+		realpath = path
+	} else if strings.HasPrefix(path, "~") {
+		tmp := strings.TrimPrefix(path, "~")
+		realpath = filepath.Join(homeDir, tmp)
+	} else {
+		realpath = filepath.Join(goSrcDir, path)
+		isGoLang = true
+	}
+
+	if os.Getenv("REPO_AUTO_CLONE") == "true" {
+		err := f.Clone(goSrcDir, path)
+		if err != nil {
+			// directory doesn't exist. exit with nil and error nil
+			return path, realpath, goSrcDir, false, errors.New("git clone")
+		}
+	}
+
+	if !IsDirectory(realpath) {
+		log.Log(FORGEPBWARN, "directory doesn't exist", realpath)
+		// directory doesn't exist. exit with nil and error nil
+		return path, realpath, goSrcDir, false, errors.New(realpath + " does not exist")
+	}
+
+	filename := filepath.Join(realpath, ".git/config")
+
+	_, err = os.Open(filename)
+	if err != nil {
+		// log.Log(WARN, "Error reading .git/config:", filename, err)
+		// log.Log(WARN, "TODO: find .git/config in parent directory")
+		return path, realpath, goSrcDir, false, err
+	}
+	return path, realpath, goSrcDir, isGoLang, nil
+}
+*/
+
+// TODO: make some config file for things like this
+// can be used to work around temporary problems
+func clonePathHack(dirname string, basedir string, gopath string) error {
+	// newdir = helloworld
+	// basedir = /home/jcarr/go/src/go.wit.com/apps
+	// giturl = https://gitea.wit.com/gui/helloworld
+	// func cloneActual(newdir, basedir, giturl string) error {
+
+	switch gopath {
+	case "golang.org/x/crypto":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/crypto")
+	case "golang.org/x/mod":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/mod")
+	case "golang.org/x/net":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/net")
+	case "golang.org/x/sys":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sys")
+	case "golang.org/x/sync":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sync")
+	case "golang.org/x/term":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/term")
+	case "golang.org/x/text":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/text")
+	case "golang.org/x/tools":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/tools")
+	case "golang.org/x/xerrors":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/xerrors")
+	case "google.golang.org/protobuf":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/protobuf")
+	case "google.golang.org/genproto":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/genproto")
+	case "google.golang.org/api":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/api")
+	case "google.golang.org/grpc":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/grpc")
+	case "google.golang.org/appengine":
+		return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/appengine")
+	}
+
+	return errors.New("no gopath override here")
+}
+
+// attempt to git clone if the go path doesn't exist
+// does a git clone, if it works, returns true
+// workdir = /home/jcarr/go/src/
+// gopath  = go.wit.com/apps/helloworld
+func (f *Forge) Clone(gopath string) (*gitpb.Repo, error) {
+	var err error
+	pb, err := f.Repos.NewGoPath(f.goSrc, gopath)
+	if err == nil {
+		return pb, err
+	}
+	workdir := f.goSrc
+	fullpath := filepath.Join(workdir, gopath)
+	dirname := filepath.Base(fullpath)
+
+	basedir := strings.TrimSuffix(fullpath, dirname)
+
+	url := "https://" + gopath
+	log.Info("trying git clone")
+	log.Info("gopath =", gopath)
+
+	// try a direct git clone against the gopath
+	// cloneActual("helloworld", "/home/jcarr/go/src/go.wit.com/apps", "https://go.wit.com/apps/helloworld")
+	if err = cloneActual(dirname, basedir, url); err == nil {
+		// git clone worked!
+		return f.Repos.NewGoPath(f.goSrc, gopath)
+	}
+	log.Info("direct attempt at git clone failed", url)
+
+	// if direct git clone doesn't work, look for a redirect
+	// go directly to the URL as that is autoritive. If that failes
+	// try the go package system as maybe the git site no longer exists
+	if url, err = findGoImport(url); err != nil {
+		log.Info("findGoImport() DID NOT WORK", url)
+		log.Info("findGoImport() DID NOT WORK", err)
+	} else {
+		if err := cloneActual(dirname, basedir, url); err == nil {
+			// git clone worked!
+			return f.Repos.NewGoPath(f.goSrc, gopath)
+		}
+	}
+	log.Info("git clone from 'go-import' info failed", url)
+
+	// query the golang package system for the last known location
+	// NOTE: take time to thank the go developers and google for designing this wonderful system
+	if url, err = runGoList(gopath); err != nil {
+		log.Info("go list failed", err)
+	} else {
+		if err := cloneActual(dirname, basedir, url); err == nil {
+			// git clone worked!
+			return f.Repos.NewGoPath(f.goSrc, gopath)
+		}
+	}
+	log.Info("git clone from 'git list' info failed", url)
+
+	// try to parse a redirect
+
+	if err = clonePathHack(dirname, basedir, gopath); err == nil {
+		// WTF didn't go-import or go list work?
+		return f.Repos.NewGoPath(f.goSrc, gopath)
+	}
+
+	return nil, errors.New("can not find git sources for gopath " + gopath)
+}
+
+// newdir = helloworld
+// basedir = /home/jcarr/go/src/go.wit.com/apps
+// giturl = https://gitea.wit.com/gui/helloworld
+func cloneActual(newdir, basedir, giturl string) error {
+	log.Info("cloneActual() newdir  =", newdir)
+	log.Info("cloneActual() basedir =", basedir)
+	log.Info("cloneActual() giturl  =", giturl)
+	if !IsDirectory(basedir) {
+		os.MkdirAll(basedir, 0750)
+	}
+	err := os.Chdir(basedir)
+	if err != nil {
+		log.Warn("chdir failed", basedir, err)
+		return err
+	}
+
+	cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir}
+	log.Info("Running:", cmd)
+	r := shell.PathRunRealtime(basedir, cmd)
+	if r.Error != nil {
+		log.Warn("git clone error", r.Error)
+		return r.Error
+	}
+
+	fullpath := filepath.Join(basedir, newdir)
+	if !IsDirectory(fullpath) {
+		log.Info("git clone failed", giturl)
+		return errors.New("git clone failed " + giturl)
+	}
+	gitdir := filepath.Join(fullpath, ".git")
+	if IsDirectory(gitdir) {
+		log.Info("git cloned worked to", fullpath)
+		return nil
+	}
+	// git clone didn't really work but did make a directory
+	log.Info("fullpath is probably empty", fullpath)
+	panic("crapnuts. rmdir fullpath here?")
+}
+
+// check the server for the current go path to git url mapping
+// for example:
+// This will check go.wit.com for "go.wit.com/apps/go-clone"
+// and return where the URL to do git clone should be
+func findGoImport(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	tmp := string(bodyBytes)
+	parts := strings.Split(tmp, "go-import")
+	if len(parts) < 2 {
+		return "", errors.New("missing go-import")
+	}
+	// this is terrible, it doesn't even look for 'content='
+	// but again, this is just a hack for my own code to be
+	// usuable after the removal in go v1.22 of the go get clone behavior that was in v1.21
+	parts = strings.Split(parts[1], "\"")
+	var newurl string
+	for {
+		if len(parts) == 0 {
+			break
+		}
+		tmp := strings.TrimSpace(parts[0])
+		fields := strings.Split(tmp, " ")
+		// log.Info("FIELDS:", fields)
+		if len(fields) == 3 {
+			newurl = fields[2]
+			break
+		}
+		parts = parts[1:]
+	}
+	if newurl == "" {
+		return "", errors.New("missing git content string")
+	}
+
+	return newurl, nil
+}
+
+func IsDirectory(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return info.IsDir()
+}

repoNew.go

@@ -0,0 +1,7 @@
+package forgepb
+
+import "go.wit.com/lib/protobuf/gitpb"
+
+func (f *Forge) NewGoPath(gopath string) (*gitpb.Repo, error) {
+	return f.Repos.NewGoPath(f.goSrc, gopath)
+}

repoSettings.go

@@ -14,6 +14,12 @@ import (
 	"strings"
 )
 
+/*
+func (f *Forge) SortByPath() *ForgeConfigIterator {
+	return f.Config.SortByPath()
+}
+*/
+
 func (all *ForgeConfigs) UpdateGoPath(name string, gopath string) bool {
 	oldr := all.DeleteByGoPath(name)
 	if oldr == nil {