Compare commits

..

No commits in common. "master" and "v0.0.5" have entirely different histories.

12 changed files with 197 additions and 542 deletions

2
.gitignore vendored
View File

@ -3,4 +3,4 @@ go.mod
go.sum
/files/*
/work/*
go-clone*
go-clone

View File

@ -1,81 +1,49 @@
VERSION = $(shell git describe --tags)
BUILDTIME = $(shell date +%Y.%m.%d_%H%M)
run: install
go-clone --version
run: build
@#./go-clone --work github.com/rclone/rclone
@# ./go-clone --work go.wit.com/apps/basicwindow
./go-clone go.wit.com/apps/basicwindow
# test using --no-work against ~/go/src
homeGoSrc: build
./go-clone --no-work go.wit.com/apps/basicwindow
modernc: build
./go-clone --no-work modernc.org/sqlite
vet:
@GO111MODULE=off go vet
@echo this go binary package builds okay
@echo this go library package builds okay
no-gui: install build-darwin build-windows
no-gui: build
./go-clone --no-gui
build: goimports
GO111MODULE=off go build -v \
-ldflags "-X main.VERSION=${VERSION} -X main.BUILDTIME=${BUILDTIME} -X gui.GUIVERSION=${VERSION}"
build:
GO111MODULE=off go build -v -ldflags "-X main.VERSION=${VERSION}" -ldflags "-X main.GUIVERSION=${VERSION}"
@# GO111MODULE=off go build -v -ldflags "-X main.GUIVERSION=${VERSION}"
build-darwin:
GOOS=darwin GOARCH=amd64 GO111MODULE=off go build -v -o go-clone-darwin.x86 \
-ldflags "-X main.VERSION=${VERSION} -X main.BUILDTIME=${BUILDTIME} -X gui.GUIVERSION=${VERSION}"
build-darwin-arm64:
GOOS=darwin GOARCH=arm64 GO111MODULE=off go build -v -o go-clone-darwin.arm \
-ldflags "-X main.VERSION=${VERSION} -X main.BUILDTIME=${BUILDTIME} -X gui.GUIVERSION=${VERSION}"
build-windows:
GOOS=windows GOARCH=amd64 GO111MODULE=off go build -v -o go-clone.exe \
-ldflags "-X main.VERSION=v0.7.46 -X main.BUILDTIME=2025.02.22_0643 -X gui.GUIVERSION=v0.7.46"
install: goimports
GO111MODULE=off go install \
-ldflags "-X main.VERSION=${VERSION} -X main.BUILDTIME=${BUILDTIME} -X gui.GUIVERSION=${VERSION}"
install:
GO111MODULE="off" go install -v
goimports:
reset
goimports -w *.go
# // to globally reset paths:
# // gofmt -w -r "go.wit.com/gui -> go.wit.com/gui/gui" .
redomod:
rm -f go.*
GO111MODULE= go mod init
GO111MODULE= go mod tidy
go mod edit -go=1.20
gui-gocui: build
reset:
# clear your terminal
reset
gocui: build
reset
./go-clone --gui gocui >/tmp/witgui.log.stderr 2>&1
nocui: build
nocui: reset build
./go-clone --gui nocui
clean:
rm -f go.*
-rm go-clone*
go-mod-clean purge
# this will test the golang.org/x -> googlesource override
git-clone:
./go-clone --recursive --go-src --no-work go.wit.com/lib/daemons/virtigod
test-build: build
./go-clone --build go.wit.com/apps/guireleaser
test-install: build
./go-clone --install go.wit.com/apps/guireleaser
debian:
go-deb --no-gui --repo go.wit.com/apps/go-clone
pull: build
./go-clone --dry-run --pull
pullreal: build
./go-clone --pull
fetch: build
./go-clone --dry-run --fetch
modernc: build
./go-clone --no-work --recursive modernc.org/sqlite
-rm go-clone

View File

@ -1,54 +1 @@
# go-clone
In 2018, [gohack](https://github.com/rogpeppe/gohack) was written for the same reasons
this tool was written. gohack has a good justification for this kind of tool.
## Install go-glone
go install go.wit.com/apps/go-clone@latest
## go-glone itself
This will download the sources for go-clone:
go-clone go.wit.com/apps/go-clone
go-clone works in ~/go/src unless it finds a go.work file in a parent dir
If you are using a go.work file, this will autocreate one. The old
one is saved as go.work.last
go-clone --auto-work go.wit.com/apps/go-clone
Or to recursively clone all the build dependancies:
go-clone --recursive go.wit.com/apps/go-clone
## debian packages
Debian packages are at mirrors.wit.com
## TODO:
* use protobuf
* move edge case mappings to a config file
* figure out how to detect gooogle.golang.org mapping with 'go list'
## these are notes from Gohack: mutable checkouts of Go module dependencies
The new Go module system is awesome. It ensures repeatable, deterministic
builds of Go code. External module code is cached locally in a read-only
directory, which is great for reproducibility. But if you're used to the
global mutable namespace that is `$GOPATH`, there's an obvious question:
what if I'm hacking on my program and I *want* to change one of those
external modules?
You might want to put a sneaky `log.Printf` statement to find out how
some internal data structure works, or perhaps try out a bug fix to see
if it solves your latest problem. But since all those external modules
are in read-only directories, it's hard to change them. And you really
don't want to change them anyway, because that will break the integrity
checking that the Go tool does when building.
Luckily the modules system provides a way around this: you can add a
`replace` statement to the `go.mod` file which substitutes the contents
of a directory holding a module for the readonly cached copy. You can of
course do this manually, but gohack aims to make this process pain-free.

44
argv.go
View File

@ -1,53 +1,25 @@
package main
import (
"fmt"
"os"
)
/*
this parses the command line arguements
this enables command line options from other packages like 'gui' and 'log'
*/
var argv args
type args struct {
Repo string `arg:"positional" help:"go import path"`
AutoWork bool `arg:"--work" default:"false" help:"recreate the go.work file"`
DryRun bool `arg:"--dry-run" help:"show what would be run"`
Recursive bool `arg:"--recursive" default:"true" help:"recursively clone all dependencies"`
Build bool `arg:"--build" help:"try to build it after clone"`
Install bool `arg:"--install" help:"try to install it after clone"`
Ignore bool `arg:"--ignore" default:"false" help:"ignore weird clone errors from non-standard repos"`
// Fetch bool `arg:"--git-fetch" default:"false" help:"run 'git fetch' on all your repos"`
}
func (args) Version() string {
return "go-clone " + VERSION + " Built on " + BUILDTIME
Repo string `arg:"positional" help:"go import path"`
Work bool `arg:"--work" help:"make a work directory"`
NoWork bool `arg:"--no-work" help:"does not make a go.work file"`
}
func (a args) Description() string {
return `
git clone go repositories recursively
Example usage: "go-clone go.wit.com/apps/go-clone"
Examples:
go-clone go.wit.com/apps/go-clone # 'git clone' go-clone
`
This will pull down the go sources and
the repositories in the go.sum file using git clone`
}
func (a args) DoAutoComplete(argv []string) {
switch argv[0] {
case "checkout":
fmt.Println("user devel master ")
case "--recursive":
fmt.Println("true false")
default:
if argv[0] == ARGNAME {
// list the subcommands here
fmt.Println("--dry-run --recursive --work")
}
}
os.Exit(0)
func (args) Version() string {
return "go-clone " + VERSION
}

View File

@ -1,24 +0,0 @@
package main
import (
"os"
"go.wit.com/lib/gui/shell"
"go.wit.com/log"
)
func build() error {
err := forge.Build(workingRepo, nil)
pwd, _ := os.Getwd()
if err == nil {
log.Info("this totally worked", pwd)
shell.RunEcho([]string{"ls", "-l"})
log.Info("ran ls")
} else {
log.Info("this totally did not work", pwd)
shell.RunEcho([]string{"ls", "-l"})
log.Info("ran ls")
badExit(err)
}
return err
}

196
clone.go
View File

@ -1,196 +0,0 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"go.wit.com/lib/protobuf/gitpb"
"go.wit.com/log"
)
/*
# to support distributed 'git bug'
# create a new user:
git bug user new -e "jcarr@wit.com" -n "Jeff Carr"
git pull origin +refs/bugs/\*:refs/bugs/\*
git pull origin +refs/identities/\*:refs/identities/\*
git show-ref | grep refs/bugs/
git log refs/bugs/<some-id>
git config --add remote.origin.fetch '+refs/bugs/*:refs/bugs/*'
git config --add remote.origin.fetch '+refs/identities/*:refs/identities/*'
git config --get-all remote.origin.fetch
[remote "origin"]
url = ...
fetch = +refs/heads/*:refs/remotes/origin/*
fetch = +refs/bugs/*:refs/bugs/*
fetch = +refs/identities/*:refs/identities/*
# remove the caches
rm -rf .git/git-bug
# rebuild the cache with any command
git bug user
*/
// CleanRepoURL removes http://, https://, and .git suffix from the given URL if present.
func CleanRepoURL(url string) string {
// Trim protocol prefix
if strings.HasPrefix(url, "http://") {
url = strings.TrimPrefix(url, "http://")
} else if strings.HasPrefix(url, "https://") {
url = strings.TrimPrefix(url, "https://")
}
// Trim trailing .git
url = strings.TrimSuffix(url, ".git")
return url
}
func clone(gopath string) (*gitpb.Repo, error) {
// if the user defined a repo, attempt to download it now
if gopath == "" {
// nothing to clone
// user probably wants to --recursive on current working dir
return nil, errors.New("gopath was blank")
}
gopath = CleanRepoURL(gopath)
os.Setenv("REPO_AUTO_CLONE", "true")
// pb, _ := forge.NewGoPath(gopath)
check := forge.FindAnyPath(filepath.Join(forge.Config.ReposDir, gopath))
if check != nil {
if check.IsValidDir() {
// repo already exists and is valid
return check, nil
}
}
pb, err := forge.GoClone(gopath)
if err != nil {
log.Info("clone() could not download err:", err)
return nil, err
}
autoWork()
if err := makeValidGoSum(pb); err != nil {
return nil, err
}
log.Info("go-clone clone() onward and upward")
return pb, nil
}
// really only does go.sum things
// so not 'really' recursive
// but that is because go.sum is supposed
// to have everything required in it
func recursiveClone(check *gitpb.Repo) error {
var good int
var bad int
badmap := make(map[string]error)
if check == nil {
return errors.New("repo was nil")
}
log.Info("STARTING RECURSIVE CLONE", check.Namespace)
log.Info("STARTING RECURSIVE CLONE", check.Namespace)
// if just cloned, parse the go.sum file for deps
if check.ParseGoSum() {
} else {
makeValidGoSum(check)
}
check.ReloadForce()
if check.GoDeps == nil {
log.Info("repo godeps == nil", check.Namespace)
return errors.New("no go deps?")
}
// probably this should never be 0 because GoPrimitive should have been true otherwise
if check.GoDeps.Len() == 0 {
log.Info("repo len(godeps) == 0", check.Namespace)
return errors.New("go.sum never parsed?")
}
log.Info("deps for", check.Namespace, "len()", check.GoDeps.Len())
deps := check.GoDeps.SortByGoPath()
for deps.Scan() {
depRepo := deps.Next()
log.Info("download:", depRepo.GetGoPath())
_, err := clone(depRepo.GetGoPath())
if err != nil {
log.Info("recursiveClone() could not download", depRepo.GetGoPath())
log.Info("err:", err)
bad += 1
badmap[depRepo.GetGoPath()] = err
} else {
log.Info("downloaded", depRepo.GetGoPath())
good += 1
}
}
log.Info("got", good, "repos", "failed on", bad, "repos")
if bad != 0 {
log.Info("clone() ERROR len(badmap)", len(badmap))
for gopath, err := range badmap {
log.Info("clone() ERROR", gopath, err)
}
if !argv.Ignore {
return errors.New("clone failed on some repos")
}
}
return nil
}
func makeValidGoSum(check *gitpb.Repo) error {
// attempt to grab the notes
check.RunQuiet([]string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"})
if err := check.RunVerbose([]string{"forge", "list"}); err != nil {
log.Info("")
log.Info("Do you have go-mod-clean? Otherwise:")
log.Info(" go install go.wit.com/apps/go-mod-clean@latest")
log.Info("")
}
log.Info("try running go-mod-clean")
// update go.sum and go.mod
if err := check.RunVerbose([]string{"go-mod-clean", "lax"}); err != nil {
log.Info("")
log.Info("Do you have go-mod-clean? Otherwise:")
log.Info(" go install go.wit.com/apps/go-mod-clean@latest")
log.Info("")
}
if check.ParseGoSum() {
return nil
}
// if this fails, just use go mod
if err := check.ValidGoSum(); err != nil {
cmd := []string{"go", "mod", "init", check.Namespace}
log.Info("try running", cmd)
if _, err := check.RunQuiet(cmd); err != nil {
log.Info("go mod init failed", err)
}
if _, err := check.RunQuiet([]string{"go", "mod", "tidy"}); err != nil {
log.Info("go mod tidy failed", err)
}
}
if check.ParseGoSum() {
return nil
}
return fmt.Errorf("could not make a valid go.mod")
}

View File

@ -1,8 +1,10 @@
Source: go-clone
Build-Depends: golang, protoc-gen-go, autogenpb
Build-Depends: golang
Package: go-clone
Maintainer: Jeff Carr <jcarr@wit.com>
Architecture: amd64
Depends:
Recommends: go-clone
Description: 'git clone' the sources for a go project
Description: git clones a go package and it's dependancies
does GO111MODULE=auto go get -v (more or less) only works on git
repositories. This is an experiment. GO should be used instead.

174
main.go
View File

@ -1,91 +1,135 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"go.wit.com/dev/alexflint/arg"
"go.wit.com/lib/gui/prep"
"go.wit.com/lib/protobuf/forgepb"
"go.wit.com/lib/protobuf/gitpb"
"go.wit.com/gui"
"go.wit.com/lib/gui/repolist"
"go.wit.com/lib/gui/repostatus"
"go.wit.com/lib/gui/shell"
"go.wit.com/log"
)
// sent via -ldflags
var VERSION string
var BUILDTIME string
var ARGNAME string = "go-clone"
var pp *arg.Parser
var forge *forgepb.Forge
var workingRepo *gitpb.Repo
var rv *repolist.RepoList
var myargs args
func main() {
log.Info("go-clone version", VERSION, "built on", BUILDTIME)
// command line parsing & handling
prep.Bash(ARGNAME, argv.DoAutoComplete) // todo: make this: prep.Bash(argv)
arg.MustParse(&myargs)
pp = arg.MustParse(&argv)
if myargs.Repo == "" {
// tmp.WriteHelp(os.Stdout)
// fmt.Println("hello world")
tmp := myargs.Description()
fmt.Println(tmp)
fmt.Println(myargs.Version())
os.Exit(0)
}
forge = forgepb.Init()
var err error
// attempt to clone, returns *gitpb.Repo
workingRepo, err = clone(argv.Repo)
wdir, err := findWorkFile()
if err != nil {
badExit(err)
log.Info(err)
os.Exit(-1)
}
if argv.Recursive {
log.Info("STARTING RECURSIVE CLONE", workingRepo.GetGoPath())
if err := recursiveClone(workingRepo); err != nil {
badExit(err)
log.Info("scanning directory:", wdir)
os.Setenv("REPO_WORK_PATH", wdir)
// readControlFile()
b := gui.RawBox()
rv = repolist.AutotypistView(b)
os.Setenv("REPO_AUTO_CLONE", "true")
newr, err := rv.NewRepo(myargs.Repo)
if err != nil {
log.Info("could not download:", err)
os.Exit(-1)
}
newr.Status.MakeRedomod()
// rv.NewRepo("go.wit.com/apps/helloworld")
for _, path := range repostatus.ScanGitDirectories(wdir) {
gopath := strings.TrimPrefix(path, wdir)
gopath = strings.Trim(gopath, "/")
// log.Info("Also should add:", gopath)
rv.NewRepo(gopath)
}
godep := newr.Status.GetGoDeps()
for gopath, version := range godep {
if _, err := rv.NewRepo(gopath); err != nil {
log.Info("git clone failed for", gopath, version)
}
}
autoWork()
if argv.Build {
log.Info("STARTING BUILD", workingRepo.GetGoPath())
/*
if err := makeValidGoSum(workingRepo); err != nil {
badExit(err)
}
if err := build(); err != nil {
badExit(err)
}
*/
if workingRepo.GetRepoType() == "binary" || workingRepo.GetRepoType() == "plugin" {
log.Info("build will probably fail", workingRepo.GetGoPath(), "is", workingRepo.GetRepoType())
}
if err := forge.Build(workingRepo, nil); err != nil {
log.Warn("BUILD FAILED", workingRepo.GetGoPath(), err)
badExit(err)
}
log.Info("all repositories are cloned")
if ! myargs.NoWork {
log.Info("Creating", wdir + "/go.work")
rv.MakeGoWork()
}
if argv.Install {
log.Info("STARTING INSTALL", workingRepo.GetGoPath())
// can only install binary or plugin go packages
if workingRepo.GetRepoType() == "binary" || workingRepo.GetRepoType() == "plugin" {
log.Info("install will probably fail", workingRepo.GetGoPath(), "is", workingRepo.GetRepoType())
}
if err := forge.Install(workingRepo, nil); err != nil {
log.Warn("INSTALL FAILED", workingRepo.GetGoPath(), err)
badExit(err)
}
}
okExit("")
}
func okExit(thing string) {
if thing != "" {
log.Info(thing, "ok")
// look for or make a go.work file
// otherwise use ~/go/src
func findWorkFile() (string, error) {
pwd, err := os.Getwd()
if err == nil {
// Check for go.work in the current directory and then move up until root
if pwd, err := digup(pwd); err == nil {
// found an existing go.work file
os.Chdir(pwd)
return pwd, nil
}
// if the user added '--work' on the cmdline, make a work directory and init the go.work file
if myargs.Work {
pwd, err = os.Getwd()
newpwd := filepath.Join(pwd, "work")
shell.Mkdir(newpwd)
os.Chdir(newpwd)
if _, err := os.Stat("go.work"); err == nil {
return newpwd, nil
}
shell.RunPath(newpwd, []string{"go", "work", "init"})
if shell.Exists("go.work") {
return newpwd, nil
}
}
}
log.Info("Finished clone on", workingRepo.GetGoPath(), "ok")
forge.ConfigSave()
os.Exit(0)
// there are no go.work files, resume the ~/go/src behavior from prior to golang 1.22
// this is the 'old way" and works fine for me. I use it because I like the ~/go/src directory
// because I know exactly what is in it: GO stuff & nothing else
homeDir, err := os.UserHomeDir()
if err != nil {
return "", err
}
pwd = filepath.Join(homeDir, "go/src")
shell.Mkdir(pwd)
os.Chdir(pwd)
return pwd, nil
}
func badExit(err error) {
log.Info("Total repositories:", forge.Repos.Len())
log.Info("Finished go-clone with error", err, forge.Config.ReposDir)
os.Exit(-1)
func digup(path string) (string, error) {
for {
workFilePath := filepath.Join(path, "go.work")
if _, err := os.Stat(workFilePath); err == nil {
return path, nil // Found the go.work file
} else if !os.IsNotExist(err) {
return "", err // An error other than not existing
}
parentPath := filepath.Dir(path)
if parentPath == path {
break // Reached the filesystem root
}
path = parentPath
}
return "", fmt.Errorf("no go.work file found")
}

49
readWorkFile.go Normal file
View File

@ -0,0 +1,49 @@
package main
import (
"bufio"
"os"
"path/filepath"
"strings"
"go.wit.com/lib/gui/shell"
"go.wit.com/log"
)
func addDir(d string) {
if shell.IsDir(d) {
rv.NewRepo(d)
}
}
func readControlFile(path string) error {
fullname := filepath.Join(path, "go.work")
file, err := os.Open(fullname)
if err != nil {
return err
}
defer file.Close()
// pairs := make(map[string]string)
// var key string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
line = strings.TrimSpace(line)
partsNew := strings.SplitN(line, ":", 2)
if len(partsNew) > 1 {
continue
}
log.Info(line)
addDir(line)
}
if err := scanner.Err(); err != nil {
return err
}
return nil
}

View File

@ -1,30 +0,0 @@
# sometimes things go wrong temporarily. work around those here
case "golang.org/x/crypto":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/crypto")
case "golang.org/x/mod":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/mod")
case "golang.org/x/net":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/net")
case "golang.org/x/sys":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/sys")
case "golang.org/x/sync":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/sync")
case "golang.org/x/term":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/term")
case "golang.org/x/text":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/text")
case "golang.org/x/tools":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/tools")
case "golang.org/x/xerrors":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/xerrors")
case "google.golang.org/protobuf":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/protobuf")
case "google.golang.org/genproto":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/genproto")
case "google.golang.org/api":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/api")
case "google.golang.org/grpc":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/grpc")
case "google.golang.org/appengine":
return cloneActual(dirname, basedir, "https://" + "go.googlesource.com/appengine")

View File

@ -1,57 +0,0 @@
package main
import (
"bufio"
"errors"
"fmt"
"os"
"strings"
"go.wit.com/log"
)
func showOptions(b bool, s []string) {
fmt.Println("")
for _, line := range s {
fmt.Println(line)
}
fmt.Println("")
if b {
fmt.Println("Enter (Y/n)")
} else {
fmt.Println("Enter (y/N)")
}
}
// if b == true, default is to continue with 'Y'
func simpleStdin(b bool, s []string) {
/*
if argv.Auto {
return
}
*/
err := errors.New("user cancelled via stdin")
showOptions(b, s)
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
s := scanner.Text()
s = strings.TrimSpace(s)
s = strings.ToLower(s)
switch s {
case "y":
log.Info("got y")
return
case "n":
log.Info("got n")
badExit(err)
case "":
if b {
return
} else {
badExit(err)
}
default:
badExit(err)
}
}
}

20
work.go
View File

@ -1,20 +0,0 @@
package main
import (
"go.wit.com/lib/gui/shell"
"go.wit.com/log"
)
func autoWork() {
// remake the go.work file
if !argv.AutoWork {
return
}
log.Info("About to re-create", forge.Config.ReposDir+"/go.work")
shell.PathRun(forge.Config.ReposDir, []string{"mv", "go.work", "go.work.last"})
forge.MakeGoWork()
shell.PathRun(forge.Config.ReposDir, []string{"go", "work", "use"})
log.Info("")
log.Info("original go.work file saved as go.work.last")
log.Info("")
}