jcarr
/
gitpb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

smarter and faster mtime logic

This commit is contained in:
Jeff Carr 2024-12-17 00:00:49 -06:00
parent a115ba144b
commit c53da5a9a1
8 changed files with 259 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
age.go Normal file
View File

@ -0,0 +1,70 @@
package gitpb
// functions that check the ages of files
// and track if the repo needs to be re-scanned
import (
"errors"
"os"
"path/filepath"
"time"
"go.wit.com/log"
)
func (repo *Repo) LastGitPull() (time.Time, error) {
return repo.oldMtime(".git/FETCH_HEAD")
}
func (repo *Repo) GoSumAge() (time.Duration, error) {
var mtime time.Time
var err error
mtime, err = repo.oldMtime("go.sum")
if err == nil {
return time.Since(mtime), nil
}
mtime, err = repo.oldMtime("go.mod")
if err == nil {
return time.Since(mtime), nil
}
now := time.Now()
return time.Since(now), errors.New(repo.GoPath + " go.mod missing")
}
func (repo *Repo) GitChanged() bool {
fullfile := filepath.Join(repo.FullPath, ".git/FETCH_HEAD")
lasttime, err := repo.LastGitPull()
if err == nil {
// if error, something is wrong, assume true
log.Info("gitpb:", fullfile, "changed")
return true
}
newtime := repo.LastPull.AsTime()
if lasttime == newtime {
return false
}
log.Info("gitpb:", fullfile, "changed")
return true
}
func (repo *Repo) GitPullAge() time.Duration {
lastpull, err := repo.LastGitPull()
if err == nil {
// if error, something is wrong, assume true
ltime := repo.LastPull.AsTime()
return time.Since(ltime)
}
return time.Since(lastpull)
}
func (repo *Repo) oldMtime(filename string) (time.Time, error) {
pathf := filepath.Join(repo.FullPath, filename)
statf, err := os.Stat(pathf)
if err == nil {
return statf.ModTime(), nil
}
log.Log(GITPBWARN, "Mtime() os.Stat() error", pathf, err)
return time.Now(), err
}

97
changed.go Normal file
View File

@ -0,0 +1,97 @@
package gitpb
// An app to submit patches for the 30 GO GUI repos
import (
"fmt"
"time"
"go.wit.com/lib/gui/shell"
"go.wit.com/log"
"google.golang.org/protobuf/types/known/timestamppb"
)
func (repo *Repo) Mtime(fname string) *time.Time {
var fileTime *time.Time
tmp, err := repo.oldMtime(fname)
fileTime = &tmp
if err != nil {
log.Info("MTime got err", err)
return nil
}
return fileTime
}
func (repo *Repo) changedDir() bool {
fname := ".git"
fileTime := repo.Mtime(fname)
if fileTime == nil {
// .git doesn't exist. something is wrong. rescan this repo
return true
}
pbtime := repo.Times.MtimeDir
mtime := timestamppb.New(*fileTime)
if (pbtime.Seconds == mtime.Seconds) && (pbtime.Nanos == mtime.Nanos) {
return false
}
dur := mtime.AsTime().Sub(pbtime.AsTime())
repo.StateChange = fmt.Sprintf("%s changed %s", fname, shell.FormatDuration(dur))
repo.Times.MtimeDir = mtime
return true
}
func (repo *Repo) changedHead() bool {
fname := ".git/HEAD"
fileTime := repo.Mtime(fname)
if fileTime == nil {
// .git/HEAD doesn't exist. something is wrong. rescan this repo
return true
}
pbtime := repo.Times.MtimeHead
mtime := timestamppb.New(*fileTime)
if (pbtime.Seconds == mtime.Seconds) && (pbtime.Nanos == mtime.Nanos) {
return false
}
dur := mtime.AsTime().Sub(pbtime.AsTime())
repo.StateChange = fmt.Sprintf("%s changed %s", fname, shell.FormatDuration(dur))
repo.Times.MtimeHead = mtime
return true
}
func (repo *Repo) changedIndex() bool {
fname := ".git/index"
fileTime := repo.Mtime(fname)
if fileTime == nil {
// .git/index doesn't exist. something is wrong. rescan this repo
return true
}
pbtime := repo.Times.MtimeIndex
mtime := timestamppb.New(*fileTime)
if (pbtime.Seconds == mtime.Seconds) && (pbtime.Nanos == mtime.Nanos) {
return false
}
dur := mtime.AsTime().Sub(pbtime.AsTime())
repo.StateChange = fmt.Sprintf("%s changed %s", fname, shell.FormatDuration(dur))
repo.Times.MtimeIndex = mtime
return true
}
func (repo *Repo) RepoChanged() bool {
var changed bool
if repo.Times == nil {
repo.Times = new(GitTimes)
log.Info(repo.FullPath, "repo.Times were nil")
}
if repo.changedHead() {
changed = true
}
if repo.changedIndex() {
changed = true
}
if repo.changedDir() {
changed = true
}
return changed
}

24
goDep.redoGoMod.go
View File

@ -4,7 +4,6 @@ package gitpb
import (
"errors"
"time"
)
// checks to see if the go.sum and go.mod files exist
@ -24,7 +23,7 @@ func (repo *Repo) ValidGoSum() error {
}
/*
// todo: fix this
mtime, err := repo.mtime("go.mod")
mtime, err := repo.Mtime("go.mod")
if err == nil {
return err
}
@ -38,7 +37,7 @@ func (repo *Repo) ValidGoSum() error {
return errors.New("ValidGoSum() go.sum is missing")
}
/*
mtime, err := repo.mtime("go.sum")
mtime, err := repo.Mtime("go.sum")
// todo: fix this
if err == nil {
return err
@ -56,22 +55,3 @@ func (repo *Repo) GoDepsLen() int {
}
return len(repo.GoDeps.GoDeps)
}
func (repo *Repo) LastGitPull() (time.Time, error) {
return repo.mtime(".git/FETCH_HEAD")
}
func (repo *Repo) GoSumAge() (time.Duration, error) {
var mtime time.Time
var err error
mtime, err = repo.mtime("go.sum")
if err == nil {
return time.Since(mtime), nil
}
mtime, err = repo.mtime("go.mod")
if err == nil {
return time.Since(mtime), nil
}
now := time.Now()
return time.Since(now), errors.New(repo.GoPath + " go.mod missing")
}

39
reload.go Normal file
View File

@ -0,0 +1,39 @@
package gitpb
func (repo *Repo) Reload() error {
repo.Tags = new(GitTags)
repo.UpdateGitTags()
repo.GoDeps = new(GoDeps)
repo.ParseGoSum()
if repo.GoInfo != nil {
repo.ReloadGo()
}
return nil
}
func (repo *Repo) ReloadGo() error {
repo.GoPlugin = false
repo.GoProtobuf = false
repo.GoLibrary = false
repo.GoBinary = false
switch repo.goListRepoType() {
case "plugin":
repo.GoPlugin = true
case "protobuf":
repo.GoProtobuf = true
case "library":
repo.GoLibrary = true
case "binary":
repo.GoBinary = true
}
return nil
}
func (repo *Repo) SetDevelBranchName(bname string) {
repo.DevelBranchName = bname
}
func (repo *Repo) SetUserBranchName(bname string) {
repo.UserBranchName = bname
}

78
repo.new.go
View File

@ -2,12 +2,8 @@ package gitpb
import (
"errors"
"os"
"path/filepath"
"time"
"go.wit.com/log"
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
)
// scans in a new git repo. If it detects the repo is a golang project,
@ -15,52 +11,24 @@ import (
// TODO: try adding python, rails, perl, rust, other language things?
// I probably will never have time to try that, but I'd take patches for anyone
// that might see this note and feel so inclined.
func (all *Repos) NewGoPath(basepath string, gopath string, url string) (*Repo, error) {
// todo: use Repos.Lock() ?
func (all *Repos) NewGoRepo(fullpath string, gopath string) (*Repo, error) {
if gopath == "" {
return nil, errors.New("blank gopath")
}
if r := all.FindByGoPath(gopath); r != nil {
log.Info("gitpb.NewGoPath() already has gopath", r.GoPath)
log.Info("gitpb.NewGoPath() already has FullPath", r.FullPath)
log.Info("gitpb.NewGoPath() already has URL", r.URL)
// already had this gopath
return r, errors.New("gitpb.NewGoPath() duplicate gopath " + gopath)
}
log.Info("gitpb.NewGoPath() Attempting to add new path", basepath, gopath)
// if .git doesn't exist, error out here
gitpath := filepath.Join(basepath, gopath, ".git")
_, err := os.Stat(gitpath)
if err != nil {
log.Warn("gitpb.NewGoPath() not a git directory", gitpath)
return nil, err
}
// add a new one here
newr := Repo{
FullPath: filepath.Join(basepath, gopath),
GoPath: gopath,
URL: url,
}
newr.Tags = new(GitTags)
newr.UpdateGitTags()
newr.GoDeps = new(GoDeps)
switch newr.goListRepoType() {
case "plugin":
newr.GoPlugin = true
case "protobuf":
newr.GoProtobuf = true
case "library":
newr.GoLibrary = true
case "binary":
newr.GoBinary = true
}
lastpull, err := newr.LastGitPull()
if err == nil {
newr.LastPull = timestamppb.New(lastpull)
FullPath: fullpath,
}
newr.GoInfo = new(GoInfo)
newr.GoInfo.GoPath = gopath
if all.AppendUniqueGoPath(&newr) {
// worked
@ -75,39 +43,3 @@ func (all *Repos) NewGoPath(basepath string, gopath string, url string) (*Repo,
// todo: use Repos.Lock()
return nil, errors.New("repo gitpb.NewGoPath() should never have gotten here " + gopath)
}
func (repo *Repo) SetDevelBranchName(bname string) {
repo.DevelBranchName = bname
}
func (repo *Repo) SetUserBranchName(bname string) {
repo.UserBranchName = bname
}
func (repo *Repo) GitChanged() bool {
fullfile := filepath.Join(repo.FullPath, ".git/FETCH_HEAD")
lasttime, err := repo.LastGitPull()
if err == nil {
// if error, something is wrong, assume true
log.Info("gitpb:", fullfile, "changed")
return true
}
newtime := repo.LastPull.AsTime()
if lasttime == newtime {
return false
}
log.Info("gitpb:", fullfile, "changed")
return true
}
func (repo *Repo) GitPullAge() time.Duration {
lastpull, err := repo.LastGitPull()
if err == nil {
// if error, something is wrong, assume true
ltime := repo.LastPull.AsTime()
return time.Since(ltime)
}
return time.Since(lastpull)
}

31
repo.proto
View File

@ -34,6 +34,11 @@ message Repo { // `autogenpb:marshal`
string desc = 20; // what is this repo?
bytes goMod = 21; // the last go.mod file
bytes goSum = 22; // the last go.sum file
google.protobuf.Timestamp mtimeGitDir = 23; // mtime for ./git
google.protobuf.Timestamp mtimeGitHead = 24; // mtime for ./git/HEAD // these two mtimes allow really fast checks to see if git has changed
GitTimes times = 25; // store all the mtime values here. these are temporary
GoInfo goInfo = 26; // put all the go specifcs here
string stateChange = 27; // reason for state change
}
message Repos { // `autogenpb:marshal`
@ -41,3 +46,29 @@ message Repos { // `autogenpb:marshal`
string version = 2; // maybe can be used for protobuf schema change violations
repeated Repo repos = 3;
}
// should it be done this way?
message GitTimes {
google.protobuf.Timestamp lastPull = 1; // last time a git pull was done
google.protobuf.Timestamp lastUpdate = 2; // when was ReloadGit() last done
google.protobuf.Timestamp lastDirty = 3; // last time CheckDirty() was run
google.protobuf.Timestamp mtimeDir = 4; // mtime for ./git // maybe useful to track
google.protobuf.Timestamp mtimeHead = 5; // mtime for ./git/HEAD // these two mtimes allow really fast checks to see if git has changed
google.protobuf.Timestamp mtimeIndex = 6; // mtime for ./git/HEAD // probably always in sync with HEAD
google.protobuf.Timestamp mtimeFetch = 7; // mtime for ./git/FETCH_HEAD // last time 'git fetch' or 'git pull' was run on current branch?
}
// this is probably better. think about moving to this instead
message GoInfo {
string goPath = 1; // the logical path as used by golang: 'go.wit.com/apps/helloworld'
string desc = 2; // what is this repo?
bool goLibrary = 3; // is this a golang library?
bool goBinary = 4; // is this a golang binary?
bool goPrimitive = 5; // if this is a golang primitive (only has go.mod)
bool goPlugin = 6; // is this a golang plugin?
bool goProtobuf = 7; // autogen go files from .proto
GoDeps goDeps = 8; // what is in the go.sum file
GoDeps published = 9; // the last published go.mod/go.sum
bytes goMod = 10; // the last go.mod file
bytes goSum = 11; // the last go.sum file
}

15
rill.go
View File

@ -102,3 +102,18 @@ func (all *Repos) RillGitPull(part1 int, part2 int) map[*Repo]cmd.Status {
return allerr
}
func (repo *Repo) GitPullRealtime() cmd.Status {
currentName := repo.GetCurrentBranchName()
if repo.IsOnlyLocalTag(currentName) {
var result cmd.Status
result.Exit = 21
result.Error = ErrorGitPullOnLocal
// log.Info("git pull skipped on local only branch", repo.GoPath)
return result
}
var cmd []string
cmd = append(cmd, "git", "pull")
r := repo.RunRealtime(cmd)
return r
}

11
shell.go
View File

@ -6,7 +6,6 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/go-cmd/cmd"
"go.wit.com/lib/gui/shell"
@ -109,13 +108,3 @@ func (repo *Repo) IsDirectory() bool {
}
return info.IsDir()
}
func (repo *Repo) mtime(filename string) (time.Time, error) {
pathf := filepath.Join(repo.FullPath, filename)
statf, err := os.Stat(pathf)
if err == nil {
return statf.ModTime(), nil
}
log.Log(GITPB, "mtime() os.Stat() error", pathf, err)
return time.Now(), err
}