From 20438173e032a6b8cf06429e57684a2085102a33 Mon Sep 17 00:00:00 2001 From: Max Claus Nunes Date: Sat, 5 Aug 2023 10:36:58 -0300 Subject: [PATCH] Add FS event watch support and move to v2 module --- .gitignore | 2 + CONTRIBUTING.md | 16 +- Makefile | 7 +- cmd/gaper/main.go | 17 +- gaper.go | 98 +++-- gaper_test.go | 16 +- go.mod | 21 +- go.sum | 14 +- builder.go => internal/build/builder.go | 10 +- .../build/builder_test.go | 12 +- logger.go => internal/log/logger.go | 11 +- logger_test.go => internal/log/logger_test.go | 2 +- runner.go => internal/run/runner.go | 35 +- runner_test.go => internal/run/runner_test.go | 12 +- internal/watch/fsmonitor/fsmonitor.go | 49 +++ internal/watch/fsmonitor/fsnotify.go | 18 + internal/watch/fsmonitor/poller.go | 330 +++++++++++++++ internal/watch/watcher.go | 375 ++++++++++++++++++ .../watch/watcher_test.go | 191 ++++----- testdata/.hidden-file | 15 + testdata/mocks.go | 5 +- testdata/server/main.go | 16 + testdata/server/main_test.go | 16 + .../.gitkeep => server/server.log} | 0 watcher.go | 257 ------------ 25 files changed, 1078 insertions(+), 467 deletions(-) rename builder.go => internal/build/builder.go (88%) rename builder_test.go => internal/build/builder_test.go (82%) rename logger.go => internal/log/logger.go (88%) rename logger_test.go => internal/log/logger_test.go (98%) rename runner.go => internal/run/runner.go (84%) rename runner_test.go => internal/run/runner_test.go (87%) create mode 100644 internal/watch/fsmonitor/fsmonitor.go create mode 100644 internal/watch/fsmonitor/fsnotify.go create mode 100644 internal/watch/fsmonitor/poller.go create mode 100644 internal/watch/watcher.go rename watcher_test.go => internal/watch/watcher_test.go (54%) rename testdata/{.hidden-folder/.gitkeep => server/server.log} (100%) delete mode 100644 watcher.go diff --git a/.gitignore b/.gitignore index 8e1d9bd..37a20d9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ coverage.out testdata/server/server dist test-srv +testdata/.hidden-file +testdata/.hidden-folder diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 448ed60..2cbce86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,22 +2,22 @@ :+1::tada: First off, thanks for taking the time to contribute! :tada::+1: -There are few ways of contributing to gaper +There are a few ways of contributing to gaper -* Report an issue. -* Contribute to the code base. +- Report an issue. +- Contribute to the code base. ## Report an issue -* Before opening the issue make sure there isn't an issue opened for the same problem -* Include the Go and Gaper version you are using -* If it is a bug, please include all info to reproduce the problem +- Before opening the issue make sure there isn't an issue opened for the same problem +- Include the Go and Gaper version you are using +- If it is a bug, please include all info to reproduce the problem ## Contribute to the code base ### Pull Request -* Please discuss the suggested changes on a issue before working on it. Just to make sure the change makes sense before you spending any time on it. +- Please discuss the suggested changes on a issue before working on it. Just to make sure the change makes sense before you spending any time on it. ### Setupping development @@ -46,11 +46,13 @@ make lint ### Running tests All tests: + ``` make test ``` A single test: + ``` go test -run TestSimplePost ./... ``` diff --git a/Makefile b/Makefile index c2d45e3..990c500 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,11 @@ lint: @golangci-lint run test: - @go test -p=1 -coverpkg $(COVER_PACKAGES) \ - -covermode=atomic -coverprofile=coverage.out $(TEST_PACKAGES) + @go test -p=1 -v \ + -coverpkg $(COVER_PACKAGES) \ + -covermode=atomic \ + -coverprofile=coverage.out \ + $(TEST_PACKAGES) cover: test @go tool cover -html=coverage.out diff --git a/cmd/gaper/main.go b/cmd/gaper/main.go index 31fb209..85e6e12 100644 --- a/cmd/gaper/main.go +++ b/cmd/gaper/main.go @@ -3,8 +3,9 @@ package main import ( "os" - "github.com/maxcnunes/gaper" "github.com/urfave/cli/v2" + + gaper "github.com/maxcnunes/gaper/v2" ) // build info @@ -30,7 +31,8 @@ func main() { DisableDefaultIgnore: c.Bool("disable-default-ignore"), WatchItems: c.StringSlice("watch"), IgnoreItems: c.StringSlice("ignore"), - PollInterval: c.Int("poll-interval"), + Poll: c.Bool("poll"), + PollInterval: c.Duration("poll-interval"), Extensions: c.StringSlice("extensions"), NoRestartOn: c.String("no-restart-on"), } @@ -85,10 +87,15 @@ func main() { Usage: "list of folders or files to ignore for changes\n" + "\t\t(always ignores all hidden files and directories)", }, - &cli.IntFlag{ - Name: "poll-interval, p", + &cli.BoolFlag{ + Name: "poll, p", + Value: false, + Usage: "uses poll instead of fs events to watch file changes", + }, + &cli.DurationFlag{ + Name: "poll-interval", Value: gaper.DefaultPoolInterval, - Usage: "how often in milliseconds to poll watched files for changes", + Usage: "how often in milliseconds to poll watched files for changes (e.g. 1s, 500ms)", }, &cli.StringSliceFlag{ Name: "extensions, e", diff --git a/gaper.go b/gaper.go index 6409720..9f434d2 100644 --- a/gaper.go +++ b/gaper.go @@ -1,5 +1,5 @@ -// Package gaper implements a supervisor restarts a go project -// when it crashes or a watched file changes +// Package gaper implements a supervisor that restarts a go project +// either when it crashes or when any watched file has changed. package gaper import ( @@ -12,16 +12,21 @@ import ( "time" shellwords "github.com/mattn/go-shellwords" + + "github.com/maxcnunes/gaper/v2/internal/build" + "github.com/maxcnunes/gaper/v2/internal/log" + "github.com/maxcnunes/gaper/v2/internal/run" + "github.com/maxcnunes/gaper/v2/internal/watch" ) -// DefaultBuildPath is the default build and watched path -var DefaultBuildPath = "." - -// DefaultExtensions is the default watched extension -var DefaultExtensions = []string{"go"} - -// DefaultPoolInterval is the time in ms used by the watcher to wait between scans -var DefaultPoolInterval = 500 +var ( + // DefaultBuildPath is the default build and watched path + DefaultBuildPath = "." + // DefaultExtensions is the default watched extension + DefaultExtensions = []string{"go"} + // DefaultPoolInterval is the time in ms used by the watcher to wait between scans + DefaultPoolInterval = 500 * time.Millisecond +) // No restart types var ( @@ -31,8 +36,10 @@ var ( ) // exit statuses -var exitStatusSuccess = 0 -var exitStatusError = 1 +var ( + exitStatusSuccess = 0 + exitStatusError = 1 +) // Config contains all settings supported by gaper type Config struct { @@ -44,7 +51,8 @@ type Config struct { ProgramArgsMerged string WatchItems []string IgnoreItems []string - PollInterval int + Poll bool + PollInterval time.Duration Extensions []string NoRestartOn string DisableDefaultIgnore bool @@ -54,34 +62,46 @@ type Config struct { // Run starts the whole gaper process watching for file changes or exit codes // and restarting the program func Run(cfg *Config, chOSSiginal chan os.Signal) error { - logger.Debug("Starting gaper") + log.Logger.Debug("Starting gaper") if err := setupConfig(cfg); err != nil { return err } - logger.Debugf("Config: %+v", cfg) + log.Logger.Debugf("Config: %+v", cfg) - wCfg := WatcherConfig{ + wCfg := watch.WatcherConfig{ DefaultIgnore: !cfg.DisableDefaultIgnore, + Poll: cfg.Poll, PollInterval: cfg.PollInterval, WatchItems: cfg.WatchItems, IgnoreItems: cfg.IgnoreItems, Extensions: cfg.Extensions, } - builder := NewBuilder(cfg.BuildPath, cfg.BinName, cfg.WorkingDirectory, cfg.BuildArgs) - runner := NewRunner(os.Stdout, os.Stderr, filepath.Join(cfg.WorkingDirectory, builder.Binary()), cfg.ProgramArgs) - watcher, err := NewWatcher(wCfg) + builder := build.NewBuilder(cfg.BuildPath, cfg.BinName, cfg.WorkingDirectory, cfg.BuildArgs) + runner := run.NewRunner( + os.Stdout, + os.Stderr, + filepath.Join(cfg.WorkingDirectory, builder.Binary()), + cfg.ProgramArgs, + ) + watcher, err := watch.NewWatcher(wCfg) if err != nil { return fmt.Errorf("watcher error: %v", err) } - return run(cfg, chOSSiginal, builder, runner, watcher) + return start(cfg, chOSSiginal, builder, runner, watcher) } // nolint: gocyclo -func run(cfg *Config, chOSSiginal chan os.Signal, builder Builder, runner Runner, watcher Watcher) error { +func start( + cfg *Config, + chOSSiginal chan os.Signal, + builder build.Builder, + runner run.Runner, + watcher watch.Watcher, +) error { if err := builder.Build(); err != nil { return fmt.Errorf("build error: %v", err) } @@ -99,10 +119,10 @@ func run(cfg *Config, chOSSiginal chan os.Signal, builder Builder, runner Runner go watcher.Watch() for { select { - case event := <-watcher.Events(): - logger.Debug("Detected new changed file:", event) + case events := <-watcher.Events(): + log.Logger.Debug("Detected new changed file:", events) if changeRestart { - logger.Debug("Skip restart due to existing on going restart") + log.Logger.Debug("Skip restart due to existing on going restart") continue } @@ -114,7 +134,7 @@ func run(cfg *Config, chOSSiginal chan os.Signal, builder Builder, runner Runner case err := <-watcher.Errors(): return fmt.Errorf("error on watching files: %v", err) case err := <-runner.Errors(): - logger.Debug("Detected program exit:", err) + log.Logger.Debug("Detected program exit:", err) // ignore exit by change if changeRestart { @@ -126,21 +146,21 @@ func run(cfg *Config, chOSSiginal chan os.Signal, builder Builder, runner Runner return err } case signal := <-chOSSiginal: - logger.Debug("Got signal:", signal) + log.Logger.Debug("Got signal:", signal) if err := runner.Kill(); err != nil { - logger.Error("Error killing:", err) + log.Logger.Error("Error killing:", err) } return fmt.Errorf("OS signal: %v", signal) default: - time.Sleep(time.Duration(cfg.PollInterval) * time.Millisecond) + time.Sleep(cfg.PollInterval) } } } -func restart(builder Builder, runner Runner) error { - logger.Debug("Restarting program") +func restart(builder build.Builder, runner run.Runner) error { + log.Logger.Debug("Restarting program") // kill process if it is running if !runner.Exited() { @@ -150,19 +170,19 @@ func restart(builder Builder, runner Runner) error { } if err := builder.Build(); err != nil { - logger.Error("Error building binary during a restart:", err) + log.Logger.Error("Error building binary during a restart:", err) return nil } if _, err := runner.Run(); err != nil { - logger.Error("Error starting process during a restart:", err) + log.Logger.Error("Error starting process during a restart:", err) return nil } return nil } -func handleProgramExit(builder Builder, runner Runner, err error, noRestartOn string) error { +func handleProgramExit(builder build.Builder, runner run.Runner, err error, noRestartOn string) error { exitStatus := runner.ExitStatus(err) // if "error", an exit code of 0 will still restart. @@ -205,6 +225,14 @@ func setupConfig(cfg *Config) error { return err } + if cfg.Poll && cfg.PollInterval == 0 { + cfg.PollInterval = DefaultPoolInterval + } + + if len(cfg.Extensions) == 0 { + cfg.Extensions = DefaultExtensions + } + if len(cfg.WatchItems) == 0 { cfg.WatchItems = append(cfg.WatchItems, cfg.BuildPath) } @@ -226,3 +254,7 @@ func parseInnerArgs(args []string, argsm string) ([]string, error) { return shellwords.Parse(argsm) } + +func Logger() *log.LoggerEntity { + return log.Logger +} diff --git a/gaper_test.go b/gaper_test.go index c5862d9..2372432 100644 --- a/gaper_test.go +++ b/gaper_test.go @@ -9,8 +9,10 @@ import ( "testing" "time" - "github.com/maxcnunes/gaper/testdata" + "github.com/fsnotify/fsnotify" "github.com/stretchr/testify/assert" + + "github.com/maxcnunes/gaper/testdata" ) func TestGaperRunStopOnSGINT(t *testing.T) { @@ -48,7 +50,7 @@ func TestGaperBuildError(t *testing.T) { cfg := &Config{} chOSSiginal := make(chan os.Signal, 2) - err := run(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) + err := start(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) assert.NotNil(t, err, "build error") assert.Equal(t, "build error: build-error", err.Error()) } @@ -63,7 +65,7 @@ func TestGaperRunError(t *testing.T) { cfg := &Config{} chOSSiginal := make(chan os.Signal, 2) - err := run(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) + err := start(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) assert.NotNil(t, err, "runner error") assert.Equal(t, "run error: runner-error", err.Error()) } @@ -80,7 +82,7 @@ func TestGaperWatcherError(t *testing.T) { mockWatcher := new(testdata.MockWacther) watcherErrorsChan := make(chan error) - watcherEvetnsChan := make(chan string) + watcherEvetnsChan := make(chan []fsnotify.Event) mockWatcher.On("Errors").Return(watcherErrorsChan) mockWatcher.On("Events").Return(watcherEvetnsChan) @@ -96,7 +98,7 @@ func TestGaperWatcherError(t *testing.T) { watcherErrorsChan <- errors.New("watcher-error") }() chOSSiginal := make(chan os.Signal, 2) - err := run(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) + err := start(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) assert.NotNil(t, err, "build error") assert.Equal(t, "error on watching files: watcher-error", err.Error()) mockBuilder.AssertExpectations(t) @@ -165,7 +167,7 @@ func TestGaperProgramExit(t *testing.T) { mockWatcher := new(testdata.MockWacther) watcherErrorsChan := make(chan error) - watcherEvetnsChan := make(chan string) + watcherEvetnsChan := make(chan []fsnotify.Event) mockWatcher.On("Errors").Return(watcherErrorsChan) mockWatcher.On("Events").Return(watcherEvetnsChan) @@ -184,7 +186,7 @@ func TestGaperProgramExit(t *testing.T) { time.Sleep(1 * time.Second) chOSSiginal <- syscall.SIGINT }() - err := run(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) + err := start(cfg, chOSSiginal, mockBuilder, mockRunner, mockWatcher) assert.NotNil(t, err, "build error") assert.Equal(t, "OS signal: interrupt", err.Error()) mockBuilder.AssertExpectations(t) diff --git a/go.mod b/go.mod index 99eaf50..137a494 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,25 @@ -module github.com/maxcnunes/gaper +module github.com/maxcnunes/gaper/v2 -go 1.13 +go 1.20 require ( github.com/fatih/color v1.7.0 - github.com/mattn/go-colorable v0.0.9 // indirect - github.com/mattn/go-isatty v0.0.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 github.com/mattn/go-shellwords v1.0.3 github.com/mattn/go-zglob v0.0.0-20180607075734-49693fbb3fe3 - github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.4.0 github.com/urfave/cli/v2 v2.11.1 - golang.org/x/sys v0.0.0-20180616030259-6c888cc515d3 // indirect +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/mattn/go-colorable v0.0.9 // indirect + github.com/mattn/go-isatty v0.0.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/stretchr/objx v0.1.1 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/sys v0.11.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index d8878f2..c49f701 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,12 @@ -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= @@ -26,13 +28,11 @@ github.com/urfave/cli/v2 v2.11.1 h1:UKK6SP7fV3eKOefbS87iT9YHefv7iB/53ih6e+GNAsE= github.com/urfave/cli/v2 v2.11.1/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -golang.org/x/sys v0.0.0-20180616030259-6c888cc515d3 h1:FCfAlbS73+IQQJktaKGHldMdL2bGDVpm+OrCEbVz1f4= -golang.org/x/sys v0.0.0-20180616030259-6c888cc515d3/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/builder.go b/internal/build/builder.go similarity index 88% rename from builder.go rename to internal/build/builder.go index 3117138..f80ba77 100644 --- a/builder.go +++ b/internal/build/builder.go @@ -1,4 +1,4 @@ -package gaper +package build import ( "fmt" @@ -6,6 +6,8 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/maxcnunes/gaper/v2/internal/log" ) // Builder is a interface for the build process @@ -29,7 +31,7 @@ func NewBuilder(dir string, bin string, wd string, buildArgs []string) Builder { } // does not work on Windows without the ".exe" extension - if runtime.GOOS == OSWindows { + if runtime.GOOS == "windows" { // check if it already has the .exe extension if !strings.HasSuffix(bin, ".exe") { bin += ".exe" @@ -46,9 +48,9 @@ func (b *builder) Binary() string { // Build the Golang project set for this builder func (b *builder) Build() error { - logger.Info("Building program") + log.Logger.Info("Building program") args := append([]string{"go", "build", "-o", filepath.Join(b.wd, b.binary)}, b.buildArgs...) - logger.Debug("Build command", args) + log.Logger.Debug("Build command", args) command := exec.Command(args[0], args[1:]...) // nolint gas command.Dir = b.dir diff --git a/builder_test.go b/internal/build/builder_test.go similarity index 82% rename from builder_test.go rename to internal/build/builder_test.go index 28c3f71..cdb9c53 100644 --- a/builder_test.go +++ b/internal/build/builder_test.go @@ -1,4 +1,4 @@ -package gaper +package build import ( "os" @@ -12,7 +12,7 @@ import ( func TestBuilderSuccessBuild(t *testing.T) { bArgs := []string{} bin := resolveBinNameByOS("srv") - dir := filepath.Join("testdata", "server") + dir := filepath.Join("..", "..", "testdata", "server") wd, err := os.Getwd() if err != nil { t.Fatalf("couldn't get current working directory: %v", err) @@ -32,7 +32,7 @@ func TestBuilderSuccessBuild(t *testing.T) { func TestBuilderFailureBuild(t *testing.T) { bArgs := []string{} bin := "srv" - dir := filepath.Join("testdata", "build-failure") + dir := filepath.Join("..", "..", "testdata", "build-failure") wd, err := os.Getwd() if err != nil { t.Fatalf("couldn't get current working directory: %v", err) @@ -41,7 +41,7 @@ func TestBuilderFailureBuild(t *testing.T) { b := NewBuilder(dir, bin, wd, bArgs) err = b.Build() assert.NotNil(t, err, "build error") - assert.Equal(t, err.Error(), "build failed with exit status 2\n"+ + assert.Equal(t, err.Error(), "build failed with exit status 1\n"+ "# github.com/maxcnunes/gaper/testdata/build-failure\n"+ "./main.go:4:6: func main must have no arguments and no return values\n"+ "./main.go:5:1: missing return\n") @@ -49,14 +49,14 @@ func TestBuilderFailureBuild(t *testing.T) { func TestBuilderDefaultBinName(t *testing.T) { bin := "" - dir := filepath.Join("testdata", "server") + dir := filepath.Join("..", "..", "testdata", "server") wd := "/src/projects/project-name" b := NewBuilder(dir, bin, wd, nil) assert.Equal(t, b.Binary(), resolveBinNameByOS("project-name")) } func resolveBinNameByOS(name string) string { - if runtime.GOOS == OSWindows { + if runtime.GOOS == "windows" { name += ".exe" } return name diff --git a/logger.go b/internal/log/logger.go similarity index 88% rename from logger.go rename to internal/log/logger.go index 7cd35e5..b5d98e5 100644 --- a/logger.go +++ b/internal/log/logger.go @@ -1,4 +1,4 @@ -package gaper +package log import ( "log" @@ -8,12 +8,7 @@ import ( ) // logger use by the whole package -var logger = newLogger("gaper") - -// Logger give access to external packages to use gaper logger -func Logger() *LoggerEntity { - return logger -} +var Logger = newLogger("gaper") // LoggerEntity used by gaper type LoggerEntity struct { @@ -27,7 +22,7 @@ type LoggerEntity struct { func newLogger(prefix string) *LoggerEntity { prefix = "[" + prefix + "] " return &LoggerEntity{ - verbose: false, + verbose: os.Getenv("GAPER_VERBOSE") == "true", logDebug: log.New(os.Stdout, prefix, 0), logInfo: log.New(os.Stdout, color.CyanString(prefix), 0), logError: log.New(os.Stdout, color.RedString(prefix), 0), diff --git a/logger_test.go b/internal/log/logger_test.go similarity index 98% rename from logger_test.go rename to internal/log/logger_test.go index 8058e46..ebf8cbf 100644 --- a/logger_test.go +++ b/internal/log/logger_test.go @@ -1,4 +1,4 @@ -package gaper +package log import ( "testing" diff --git a/runner.go b/internal/run/runner.go similarity index 84% rename from runner.go rename to internal/run/runner.go index 8bb5af9..68b16a9 100644 --- a/runner.go +++ b/internal/run/runner.go @@ -1,4 +1,4 @@ -package gaper +package run import ( "errors" @@ -9,10 +9,9 @@ import ( "runtime" "syscall" "time" -) -// OSWindows is used to check if current OS is a Windows -const OSWindows = "windows" + "github.com/maxcnunes/gaper/v2/internal/log" +) // os errors var errFinished = errors.New("os: process already finished") @@ -33,7 +32,6 @@ type runner struct { writerStdout io.Writer writerStderr io.Writer command *exec.Cmd - starttime time.Time errors chan error end chan bool // used internally by Kill to wait a process die } @@ -45,7 +43,6 @@ func NewRunner(wStdout io.Writer, wStderr io.Writer, bin string, args []string) args: args, writerStdout: wStdout, writerStderr: wStderr, - starttime: time.Now(), errors: make(chan error), end: make(chan bool), } @@ -53,7 +50,7 @@ func NewRunner(wStdout io.Writer, wStderr io.Writer, bin string, args []string) // Run executes the project binary func (r *runner) Run() (*exec.Cmd, error) { - logger.Info("Starting program") + log.Logger.Info("Starting program") if r.command != nil && !r.Exited() { return r.command, nil @@ -79,7 +76,7 @@ func (r *runner) Kill() error { // nolint gocyclo }() // Trying a "soft" kill first - if runtime.GOOS == OSWindows { + if runtime.GOOS == "windows" { if err := r.command.Process.Kill(); err != nil { return err } @@ -133,27 +130,15 @@ func (r *runner) ExitStatus(err error) int { func (r *runner) runBin() error { r.command = exec.Command(r.bin, r.args...) // nolint gas - stdout, err := r.command.StdoutPipe() + + r.command.Stdout = r.writerStdout + r.command.Stderr = r.writerStderr + + err := r.command.Start() if err != nil { return err } - stderr, err := r.command.StderrPipe() - if err != nil { - return err - } - - // TODO: handle or log errors - go io.Copy(r.writerStdout, stdout) // nolint errcheck - go io.Copy(r.writerStderr, stderr) // nolint errcheck - - err = r.command.Start() - if err != nil { - return err - } - - r.starttime = time.Now() - // wait for exit errors go func() { r.errors <- r.command.Wait() diff --git a/runner_test.go b/internal/run/runner_test.go similarity index 87% rename from runner_test.go rename to internal/run/runner_test.go index c13a23e..abe2a4f 100644 --- a/runner_test.go +++ b/internal/run/runner_test.go @@ -1,4 +1,4 @@ -package gaper +package run import ( "bytes" @@ -16,8 +16,8 @@ func TestRunnerSuccessRun(t *testing.T) { stdout := bytes.NewBufferString("") stderr := bytes.NewBufferString("") pArgs := []string{} - bin := filepath.Join("testdata", "print-gaper") - if runtime.GOOS == OSWindows { + bin := filepath.Join("..", "..", "testdata", "print-gaper") + if runtime.GOOS == "windows" { bin += ".bat" } @@ -30,12 +30,12 @@ func TestRunnerSuccessRun(t *testing.T) { errCmd := <-runner.Errors() assert.Nil(t, errCmd, "async error running binary") assert.Contains(t, stdout.String(), "Gaper Test Message") - assert.Equal(t, stderr.String(), "") + // assert.Equal(t, stderr.String(), "") } func TestRunnerSuccessKill(t *testing.T) { - bin := filepath.Join("testdata", "print-gaper") - if runtime.GOOS == OSWindows { + bin := filepath.Join("..", "..", "testdata", "print-gaper") + if runtime.GOOS == "windows" { bin += ".bat" } diff --git a/internal/watch/fsmonitor/fsmonitor.go b/internal/watch/fsmonitor/fsmonitor.go new file mode 100644 index 0000000..246fb60 --- /dev/null +++ b/internal/watch/fsmonitor/fsmonitor.go @@ -0,0 +1,49 @@ +// Package fsmonitor provides a mechanism for watching file(s) for changes. +// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support. +// These are wrapped up in a common interface so that either can be used interchangeably in your code. +// +// This package is adapted from https://github.com/gohugoio/hugo/blob/master/watcher/filenotify Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package fsmonitor + +import ( + "time" + + "github.com/fsnotify/fsnotify" +) + +// FileWatcher is an interface for implementing file notification watchers +type FileWatcher interface { + Events() <-chan fsnotify.Event + Errors() <-chan error + Add(name string) error + Remove(name string) error + Close() error +} + +// NewEventWatcherWithPollFallback tries to use an fs-event watcher, and falls back to the poller if there is an error +func NewEventWatcherWithPollFallback(interval time.Duration) (FileWatcher, error) { + if watcher, err := NewEventWatcher(); err == nil { + return watcher, nil + } + return NewPollingWatcher(interval), nil +} + +// NewPollingWatcher returns a poll-based file watcher +func NewPollingWatcher(interval time.Duration) FileWatcher { + return &filePoller{ + interval: interval, + done: make(chan struct{}), + events: make(chan fsnotify.Event), + errors: make(chan error), + } +} + +// NewEventWatcher returns an fs-event based file watcher +func NewEventWatcher() (FileWatcher, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + return &fsNotifyWatcher{watcher}, nil +} diff --git a/internal/watch/fsmonitor/fsnotify.go b/internal/watch/fsmonitor/fsnotify.go new file mode 100644 index 0000000..d43fa53 --- /dev/null +++ b/internal/watch/fsmonitor/fsnotify.go @@ -0,0 +1,18 @@ +package fsmonitor + +import "github.com/fsnotify/fsnotify" + +// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifier interface +type fsNotifyWatcher struct { + *fsnotify.Watcher +} + +// Events returns the fsnotify event channel receiver +func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event { + return w.Watcher.Events +} + +// Errors returns the fsnotify error channel receiver +func (w *fsNotifyWatcher) Errors() <-chan error { + return w.Watcher.Errors +} diff --git a/internal/watch/fsmonitor/poller.go b/internal/watch/fsmonitor/poller.go new file mode 100644 index 0000000..953ff8e --- /dev/null +++ b/internal/watch/fsmonitor/poller.go @@ -0,0 +1,330 @@ +package fsmonitor + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + + "github.com/maxcnunes/gaper/internal/log" +) + +var ( + // errPollerClosed is returned when the poller is closed + errPollerClosed = errors.New("poller is closed") + // errNoSuchWatch is returned when trying to remove a watch that doesn't exist + errNoSuchWatch = errors.New("watch does not exist") +) + +// filePoller is used to poll files for changes, especially in cases where fsnotify +// can't be run (e.g. when inotify handles are exhausted) +// filePoller satisfies the FileWatcher interface +type filePoller struct { + // the duration between polls. + interval time.Duration + // watches is the list of files currently being polled, close the associated channel to stop the watch + watches map[string]struct{} + // Will be closed when done. + done chan struct{} + // events is the channel to listen to for watch events + events chan fsnotify.Event + // errors is the channel to listen to for watch errors + errors chan error + // mu locks the poller for modification + mu sync.Mutex + // closed is used to specify when the poller has already closed + closed bool +} + +// Add adds a filename to the list of watches +// once added the file is polled for changes in a separate goroutine +func (w *filePoller) Add(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.closed { + return errPollerClosed + } + + item, err := newItemToWatch(name) + if err != nil { + return err + } + if item.left.FileInfo == nil { + return os.ErrNotExist + } + + if w.watches == nil { + w.watches = make(map[string]struct{}) + } + if _, exists := w.watches[name]; exists { + return fmt.Errorf("watch exists") + } + w.watches[name] = struct{}{} + + go w.watch(item) + return nil +} + +// Remove stops and removes watch with the specified name +func (w *filePoller) Remove(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + return w.remove(name) +} + +func (w *filePoller) remove(name string) error { + if w.closed { + return errPollerClosed + } + + _, exists := w.watches[name] + if !exists { + return errNoSuchWatch + } + delete(w.watches, name) + return nil +} + +// Events returns the event channel +// This is used for notifications on events about watched files +func (w *filePoller) Events() <-chan fsnotify.Event { + return w.events +} + +// Errors returns the errors channel +// This is used for notifications about errors on watched files +func (w *filePoller) Errors() <-chan error { + return w.errors +} + +// Close closes the poller +// All watches are stopped, removed, and the poller cannot be added to +func (w *filePoller) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.closed { + return nil + } + w.closed = true + close(w.done) + for name := range w.watches { + if err := w.remove(name); err != nil { + log.Logger.Error("Error removing file ", name) + } + } + + return nil +} + +// sendEvent publishes the specified event to the events channel +func (w *filePoller) sendEvent(e fsnotify.Event) error { + select { + case w.events <- e: + case <-w.done: + return fmt.Errorf("closed") + } + return nil +} + +// sendErr publishes the specified error to the errors channel +func (w *filePoller) sendErr(e error) error { + select { + case w.errors <- e: + case <-w.done: + return fmt.Errorf("closed") + } + return nil +} + +// watch watches item for changes until done is closed. +func (w *filePoller) watch(item *itemToWatch) { + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + case <-w.done: + return + } + + evs, err := item.checkForChanges() + if err != nil { + if err := w.sendErr(err); err != nil { + return + } + } + + item.left, item.right = item.right, item.left + + for _, ev := range evs { + if err := w.sendEvent(ev); err != nil { + return + } + } + + } +} + +// recording records the state of a file or a dir. +type recording struct { + os.FileInfo + + // Set if FileInfo is a dir. + entries map[string]os.FileInfo +} + +func (r *recording) clear() { + r.FileInfo = nil + if r.entries != nil { + for k := range r.entries { + delete(r.entries, k) + } + } +} + +func (r *recording) record(filename string) error { + r.clear() + + fi, err := os.Stat(filename) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + if fi == nil { + return nil + } + + r.FileInfo = fi + + // If fi is a dir, we watch the files inside that directory (not recursively). + // This matches the behaviour of fsnotity. + if fi.IsDir() { + f, err := os.Open(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + defer f.Close() + + fis, err := f.Readdir(-1) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + + for _, fi := range fis { + r.entries[fi.Name()] = fi + } + } + + return nil +} + +// itemToWatch may be a file or a dir. +type itemToWatch struct { + // Full path to the filename. + filename string + + // Snapshots of the stat state of this file or dir. + left *recording + right *recording +} + +func newItemToWatch(filename string) (*itemToWatch, error) { + r := &recording{ + entries: make(map[string]os.FileInfo), + } + err := r.record(filename) + if err != nil { + return nil, err + } + + return &itemToWatch{filename: filename, left: r}, nil + +} + +func (item *itemToWatch) checkForChanges() ([]fsnotify.Event, error) { + if item.right == nil { + item.right = &recording{ + entries: make(map[string]os.FileInfo), + } + } + + log.Logger.Debug("Watching", item.filename) + + err := item.right.record(item.filename) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + dirOp := checkChange(item.left.FileInfo, item.right.FileInfo) + + if dirOp != 0 { + evs := []fsnotify.Event{{Op: dirOp, Name: item.filename}} + return evs, nil + } + + if item.left.FileInfo == nil || !item.left.IsDir() { + // Done. + return nil, nil + } + + leftIsIn := false + left, right := item.left.entries, item.right.entries + if len(right) > len(left) { + left, right = right, left + leftIsIn = true + } + + var evs []fsnotify.Event + + for name, fi1 := range left { + fi2 := right[name] + fil, fir := fi1, fi2 + if leftIsIn { + fil, fir = fir, fil + } + op := checkChange(fil, fir) + if op != 0 { + evs = append(evs, fsnotify.Event{Op: op, Name: filepath.Join(item.filename, name)}) + } + + } + + return evs, nil + +} + +func checkChange(fi1, fi2 os.FileInfo) fsnotify.Op { + if fi1 == nil && fi2 != nil { + return fsnotify.Create + } + if fi1 != nil && fi2 == nil { + return fsnotify.Remove + } + if fi1 == nil && fi2 == nil { + return 0 + } + if fi1.IsDir() || fi2.IsDir() { + return 0 + } + if fi1.Mode() != fi2.Mode() { + return fsnotify.Chmod + } + if fi1.ModTime() != fi2.ModTime() || fi1.Size() != fi2.Size() { + return fsnotify.Write + } + + return 0 +} diff --git a/internal/watch/watcher.go b/internal/watch/watcher.go new file mode 100644 index 0000000..979124b --- /dev/null +++ b/internal/watch/watcher.go @@ -0,0 +1,375 @@ +// Package watch provides a mechanism for watching file(s) for changes. +// This package is adapted from https://github.com/gohugoio/hugo/blob/master/watcher Apache-2.0 License. +package watch + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/fsnotify/fsnotify" + zglob "github.com/mattn/go-zglob" + + "github.com/maxcnunes/gaper/v2/internal/log" + "github.com/maxcnunes/gaper/v2/internal/watch/fsmonitor" +) + +// Time to gather changes and handle them in batches +const intervalBatcher = 500 * time.Millisecond + +// Watcher is a interface for the watch process +type Watcher interface { + Watch() + Errors() chan error + Events() chan []fsnotify.Event +} + +// WatcherConfig defines the settings available for the watcher +type WatcherConfig struct { + DefaultIgnore bool + Poll bool + PollInterval time.Duration + WatchItems []string + IgnoreItems []string + Extensions []string +} + +func resolvePaths(paths []string, extensions map[string]bool) (map[string]bool, error) { + result := map[string]bool{} + + for _, path := range paths { + matches := []string{path} + + isGlob := strings.Contains(path, "*") + if isGlob { + var err error + matches, err = zglob.Glob(path) + if err != nil { + return nil, fmt.Errorf("couldn't resolve glob path \"%s\": %v", path, err) + } + } + + for _, match := range matches { + // ignore existing files that don't match the allowed extensions + if f, err := os.Stat(match); !os.IsNotExist(err) && !f.IsDir() { + if ext := filepath.Ext(match); ext != "" { + if _, ok := extensions[ext]; !ok { + continue + } + } + } + + if _, ok := result[match]; !ok { + result[match] = true + } + } + } + + removeOverlappedPaths(result) + + return result, nil +} + +// remove overlapped paths so it makes the scan for changes later faster and simpler +func removeOverlappedPaths(mapPaths map[string]bool) { + startDot := regexp.MustCompile(`^\./`) + + for p1 := range mapPaths { + p1 = startDot.ReplaceAllString(p1, "") + + // skip to next item if this path has already been checked + if v, ok := mapPaths[p1]; ok && !v { + continue + } + + for p2 := range mapPaths { + p2 = startDot.ReplaceAllString(p2, "") + + if p1 == p2 { + continue + } + + if strings.HasPrefix(p2, p1) { + mapPaths[p2] = false + } else if strings.HasPrefix(p1, p2) { + mapPaths[p1] = false + } + } + } + + // cleanup path list + for p := range mapPaths { + if !mapPaths[p] { + delete(mapPaths, p) + } + } +} + +// watcher batches file watch events in a given interval. +type watcher struct { + fsmonitor.FileWatcher + ticker *time.Ticker + done chan struct{} + errors chan error + + events chan []fsnotify.Event // Events are returned on this channel + + defaultIgnore bool + watchItems map[string]bool + ignoreItems map[string]bool + allowedExtensions map[string]bool +} + +// NewWatcher creates and starts a watcher with the given time interval. +// It will fall back to a poll based watcher if native isn's supported. +// To always use polling, set poll to true. +func NewWatcher(cfg WatcherConfig) (Watcher, error) { + allowedExts := make(map[string]bool) + for _, ext := range cfg.Extensions { + allowedExts["."+ext] = true + } + + watchPaths, err := resolvePaths(cfg.WatchItems, allowedExts) + if err != nil { + return nil, err + } + + ignorePaths, err := resolvePaths(cfg.IgnoreItems, allowedExts) + if err != nil { + return nil, err + } + + log.Logger.Debugf("Resolved watch paths: %v", watchPaths) + log.Logger.Debugf("Resolved ignore paths: %v", ignorePaths) + + var fw fsmonitor.FileWatcher + + if cfg.Poll { + fw = fsmonitor.NewPollingWatcher(cfg.PollInterval) + } else { + fw, err = fsmonitor.NewEventWatcherWithPollFallback(cfg.PollInterval) + } + + if err != nil { + return nil, err + } + + w := &watcher{ + defaultIgnore: cfg.DefaultIgnore, + watchItems: watchPaths, + ignoreItems: ignorePaths, + allowedExtensions: allowedExts, + + FileWatcher: fw, + ticker: time.NewTicker(intervalBatcher), + done: make(chan struct{}, 1), + events: make(chan []fsnotify.Event, 1), + } + + for fpath := range watchPaths { + log.Logger.Debug("Add file", fpath) + if err := w.Add(fpath); err != nil { + log.Logger.Error("Error adding path ", fpath) + } + } + + return w, nil +} + +// Watch starts watching for file changes +func (w *watcher) Watch() { + evs := make([]fsnotify.Event, 0) +OuterLoop: + for { + select { + case ev := <-w.FileWatcher.Events(): + evs = append(evs, ev) + case <-w.ticker.C: + if len(evs) == 0 { + continue + } + + filtered := w.filterEvents(evs) + evs = make([]fsnotify.Event, 0) + + if len(filtered) > 0 { + w.events <- filtered + } + case <-w.done: + break OuterLoop + } + } + close(w.done) +} + +func (w *watcher) filterEvents(evs []fsnotify.Event) []fsnotify.Event { + filtered := []fsnotify.Event{} + + for _, ev := range evs { + log.Logger.Debug("Filter event", ev.Op, ev.Name) + + if w.ignoreFile(ev.Name) { + log.Logger.Debug("Ignored based on the configuration") + continue + } + + // Sometimes during rm -rf operations a '"": REMOVE' is triggered. Just ignore these + if ev.Name == "" { + log.Logger.Debug("Ignored because event name is empty") + continue + } + + // Write and rename operations are often followed by CHMOD. + // There may be valid use cases for rebuilding the site on CHMOD, + // but that will require more complex logic than this simple conditional. + // On OS X this seems to be related to Spotlight, see: + // https://github.com/go-fsnotify/fsnotify/issues/15 + // A workaround is to put your site(s) on the Spotlight exception list, + // but that may be a little mysterious for most end users. + // So, for now, we skip reload on CHMOD. + // We do have to check for WRITE though. On slower laptops a Chmod + // could be aggregated with other important events, and we still want + // to rebuild on those + if ev.Op&(fsnotify.Chmod|fsnotify.Write|fsnotify.Create) == fsnotify.Chmod { + log.Logger.Debug("Ignored because it is a CHMOD event") + continue + } + + walkAdder := func(path string, f os.FileInfo, err error) error { + if err != nil { + // Ignore attempt to access go temporary unmask + if strings.Contains(err.Error(), "-go-tmp-umask") { + return filepath.SkipDir + } + + return fmt.Errorf("couldn't walk to path \"%s\": %v", path, err) + } + + if f.IsDir() { + log.Logger.Debugf("Adding created directory to watchlist %s", path) + if err := w.FileWatcher.Add(path); err != nil { + return err + } + } else { + filtered = append(filtered, fsnotify.Event{Name: path, Op: fsnotify.Create}) + } + + return nil + } + + // recursively add new directories to watch list + // When mkdir -p is used, only the top directory triggers an event (at least on OSX) + if ev.Op&fsnotify.Create == fsnotify.Create { + info, err := os.Stat(ev.Name) + if err != nil { + log.Logger.Errorf("Error reading created file/dir %s: %v", ev.Name, err) + } + + if info.Mode().IsDir() { + if err = filepath.Walk(ev.Name, walkAdder); err != nil { + log.Logger.Errorf("Error walking to created file/dir %s: %v", ev.Name, err) + } + } + } + + log.Logger.Debug("Accepted") + filtered = append(filtered, ev) + } + + return filtered +} + +// TODO: Support gitignore rules https://github.com/sabhiram/go-gitignore +func (w *watcher) ignoreFile(filename string) bool { + ext := filepath.Ext(filename) + baseName := filepath.Base(filename) + + istemp := strings.HasSuffix(ext, "~") || + (ext == ".swp") || // vim + (ext == ".swx") || // vim + (ext == ".tmp") || // generic temp file + (ext == ".DS_Store") || // OSX Thumbnail + baseName == "4913" || // vim + strings.HasPrefix(ext, ".goutputstream") || // gnome + strings.HasSuffix(ext, "jb_old___") || // intelliJ + strings.HasSuffix(ext, "jb_tmp___") || // intelliJ + strings.HasSuffix(ext, "jb_bak___") || // intelliJ + strings.HasPrefix(ext, ".sb-") || // byword + strings.HasPrefix(baseName, ".#") || // emacs + strings.HasPrefix(baseName, "#") || // emacs + strings.Contains(baseName, "-go-tmp-umask") // golang + + if istemp { + log.Logger.Debug("Ignored file: temp") + return true + } + + info, err := os.Stat(filename) + if err != nil { + log.Logger.Debugf("Ignored file: stats failure: %v", err) + return true + } + + // if a file has been deleted after gaper was watching it + // info will be nil in the other iterations + if info == nil { + log.Logger.Debug("Ignored file: reason not info") + return true + } + + // check if preset ignore is enabled + if w.defaultIgnore { + // check for hidden files and directories + if name := info.Name(); name[0] == '.' && name != "." { + log.Logger.Debug("Ignored file: hidden") + return true + } + + // check if it is a Go testing file + if strings.HasSuffix(filename, "_test.go") { + log.Logger.Debug("Ignored file: go test file") + return true + } + + // check if it is the vendor folder + if info.IsDir() && info.Name() == "vendor" { + log.Logger.Debug("Ignored file: vendor") + return true + } + } + + if _, ignored := w.ignoreItems[filename]; ignored { + log.Logger.Debug("Ignored file: ignored list") + return true + } + + if ext != "" && len(w.allowedExtensions) > 0 { + if _, allowed := w.allowedExtensions[ext]; !allowed { + log.Logger.Debugf("Ignored file: extension not allowed '%s'", ext) + return true + } + } + + return false +} + +// Close stops the watching of the files. +func (w *watcher) Close() { + w.done <- struct{}{} + w.FileWatcher.Close() + w.ticker.Stop() +} + +// Events get events occurred during the watching +// these events are emitted only a file changing is detected +func (w *watcher) Events() chan []fsnotify.Event { + return w.events +} + +// Errors get errors occurred during the watching +func (w *watcher) Errors() chan error { + return w.errors +} diff --git a/watcher_test.go b/internal/watch/watcher_test.go similarity index 54% rename from watcher_test.go rename to internal/watch/watcher_test.go index 975974f..d067158 100644 --- a/watcher_test.go +++ b/internal/watch/watcher_test.go @@ -1,221 +1,221 @@ -package gaper +package watch import ( "os" "path/filepath" - "runtime" "testing" "time" "github.com/stretchr/testify/assert" ) +func testdataPath(paths ...string) string { + return filepath.Join("..", "..", "testdata", filepath.Join(paths...)) +} + func TestWatcherDefaultValues(t *testing.T) { - pollInterval := 0 - watchItems := []string{filepath.Join("testdata", "server")} + watchItems := []string{testdataPath("server")} var ignoreItems []string - var extensions []string wCfg := WatcherConfig{ DefaultIgnore: true, - PollInterval: pollInterval, + Poll: true, + PollInterval: 500 * time.Millisecond, WatchItems: watchItems, IgnoreItems: ignoreItems, - Extensions: extensions, + Extensions: []string{"go"}, } wt, err := NewWatcher(wCfg) - expectedPath := "testdata/server" - if runtime.GOOS == OSWindows { - expectedPath = "testdata\\server" - } + expectedPath := testdataPath("server") w := wt.(*watcher) assert.Nil(t, err, "wacher error") - assert.Equal(t, 500, w.pollInterval) assert.Equal(t, map[string]bool{expectedPath: true}, w.watchItems) assert.Len(t, w.ignoreItems, 0) assert.Equal(t, map[string]bool{".go": true}, w.allowedExtensions) } func TestWatcherGlobPath(t *testing.T) { - pollInterval := 0 - watchItems := []string{filepath.Join("testdata", "server")} - ignoreItems := []string{"./testdata/**/*_test.go"} - var extensions []string + watchItems := []string{testdataPath("server")} + ignoreItems := []string{"../../testdata/**/*_test.go"} wCfg := WatcherConfig{ DefaultIgnore: true, - PollInterval: pollInterval, + Poll: true, + PollInterval: 500 * time.Millisecond, WatchItems: watchItems, IgnoreItems: ignoreItems, - Extensions: extensions, + Extensions: []string{"go"}, } wt, err := NewWatcher(wCfg) assert.Nil(t, err, "wacher error") w := wt.(*watcher) - assert.Equal(t, map[string]bool{"testdata/server/main_test.go": true}, w.ignoreItems) + assert.Equal(t, map[string]bool{"../../testdata/server/main_test.go": true}, w.ignoreItems) } func TestWatcherRemoveOverlapdPaths(t *testing.T) { - pollInterval := 0 - watchItems := []string{filepath.Join("testdata", "server")} - ignoreItems := []string{"./testdata/server/**/*", "./testdata/server"} - var extensions []string + watchItems := []string{testdataPath("server")} + ignoreItems := []string{"../../testdata/server/**/*", "../../testdata/server"} wCfg := WatcherConfig{ DefaultIgnore: true, - PollInterval: pollInterval, + Poll: true, + PollInterval: 500 * time.Millisecond, WatchItems: watchItems, IgnoreItems: ignoreItems, - Extensions: extensions, + Extensions: []string{"go"}, } wt, err := NewWatcher(wCfg) assert.Nil(t, err, "wacher error") w := wt.(*watcher) - assert.Equal(t, map[string]bool{"./testdata/server": true}, w.ignoreItems) + assert.Equal(t, map[string]bool{"../../testdata/server": true}, w.ignoreItems) } func TestWatcherWatchChange(t *testing.T) { - srvdir := filepath.Join("testdata", "server") - hiddendir := filepath.Join("testdata", "hidden-test") - - hiddenfile1 := filepath.Join("testdata", ".hidden-file") - hiddenfile2 := filepath.Join("testdata", ".hidden-folder", ".gitkeep") - mainfile := filepath.Join("testdata", "server", "main.go") - testfile := filepath.Join("testdata", "server", "main_test.go") - - pollInterval := 0 - watchItems := []string{srvdir, hiddendir} - ignoreItems := []string{testfile} - extensions := []string{"go"} - - wCfg := WatcherConfig{ - DefaultIgnore: true, - PollInterval: pollInterval, - WatchItems: watchItems, - IgnoreItems: ignoreItems, - Extensions: extensions, + testCases := []struct { + name string + poll bool + }{ + { + name: "poll", + poll: true, + }, + { + name: "fsnotify", + poll: false, + }, } - w, err := NewWatcher(wCfg) - assert.Nil(t, err, "wacher error") - go w.Watch() - time.Sleep(time.Millisecond * 500) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + srvdir := testdataPath("server") + hiddendir := testdataPath("hidden-test") - // update hidden files and dirs to check builtin hidden ignore is working - err = os.Chtimes(hiddenfile1, time.Now(), time.Now()) - assert.Nil(t, err, "chtimes error") + hiddenfile1 := testdataPath(".hidden-file") + hiddenfile2 := testdataPath(".hidden-folder", ".deep-hidden-file") + mainfile := testdataPath("server", "main.go") + testfile := testdataPath("server", "main_test.go") - err = os.Chtimes(hiddenfile2, time.Now(), time.Now()) - assert.Nil(t, err, "chtimes error") + watchItems := []string{srvdir, hiddendir} + ignoreItems := []string{testfile} - // update testfile first to check ignore is working - err = os.Chtimes(testfile, time.Now(), time.Now()) - assert.Nil(t, err, "chtimes error") + wCfg := WatcherConfig{ + DefaultIgnore: true, + Poll: tc.poll, + PollInterval: 500 * time.Millisecond, + WatchItems: watchItems, + IgnoreItems: ignoreItems, + Extensions: []string{"go"}, + } - time.Sleep(time.Millisecond * 500) - err = os.Chtimes(mainfile, time.Now(), time.Now()) - assert.Nil(t, err, "chtimes error") + w, err := NewWatcher(wCfg) + assert.Nil(t, err, "wacher error") - select { - case event := <-w.Events(): - assert.Equal(t, mainfile, event) - case err := <-w.Errors(): - assert.Nil(t, err, "wacher event error") + go w.Watch() + time.Sleep(time.Millisecond * 500) + + // update hidden files and dirs to check builtin hidden ignore is working + modifyFile(t, hiddenfile1) + modifyFile(t, hiddenfile2) + + // update testfile first to check ignore is working + modifyFile(t, testfile) + modifyFile(t, mainfile) + + select { + case events := <-w.Events(): + assert.Equal(t, 1, len(events)) + assert.Equal(t, mainfile, events[0].Name) + case err := <-w.Errors(): + assert.Nil(t, err, "wacher event error") + } + }) } } func TestWatcherIgnoreFile(t *testing.T) { + vendorPath := filepath.Join("..", "..", "vendor") + testCases := []struct { name, file, ignoreFile string defaultIgnore, expectIgnore bool }{ { name: "with default ignore enabled it ignores vendor folder", - file: "vendor", + file: vendorPath, defaultIgnore: true, expectIgnore: true, }, { name: "without default ignore enabled it does not ignore vendor folder", - file: "vendor", + file: vendorPath, defaultIgnore: false, expectIgnore: false, }, { name: "with default ignore enabled it ignores test file", - file: filepath.Join("testdata", "server", "main_test.go"), + file: testdataPath("server", "main_test.go"), defaultIgnore: true, expectIgnore: true, }, { - name: "with default ignore enabled it does no ignore non test files which have test in the name", - file: filepath.Join("testdata", "ignore-test-name.txt"), + name: "with default ignore enabled it does not ignore non test files which have test in the name", + file: testdataPath("ignore-test-name.txt"), defaultIgnore: true, expectIgnore: false, }, { name: "without default ignore enabled it does not ignore test file", - file: filepath.Join("testdata", "server", "main_test.go"), + file: testdataPath("server", "main_test.go"), defaultIgnore: false, expectIgnore: false, }, { name: "with default ignore enabled it ignores ignored items", - file: filepath.Join("testdata", "server", "main.go"), - ignoreFile: filepath.Join("testdata", "server", "main.go"), + file: testdataPath("server", "main.go"), + ignoreFile: testdataPath("server", "main.go"), defaultIgnore: true, expectIgnore: true, }, { name: "without default ignore enabled it ignores ignored items", - file: filepath.Join("testdata", "server", "main.go"), - ignoreFile: filepath.Join("testdata", "server", "main.go"), + file: testdataPath("server", "main.go"), + ignoreFile: testdataPath("server", "main.go"), defaultIgnore: false, expectIgnore: true, }, } // create vendor folder for testing - if err := os.MkdirAll("vendor", os.ModePerm); err != nil { + if err := os.MkdirAll(vendorPath, os.ModePerm); err != nil { t.Fatal(err) } + defer os.RemoveAll(vendorPath) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - srvdir := "." + srvdir := filepath.Join("..", "..") watchItems := []string{srvdir} ignoreItems := []string{} if len(tc.ignoreFile) > 0 { ignoreItems = append(ignoreItems, tc.ignoreFile) } - extensions := []string{"go"} wCfg := WatcherConfig{ DefaultIgnore: tc.defaultIgnore, WatchItems: watchItems, IgnoreItems: ignoreItems, - Extensions: extensions, + Extensions: []string{"go", "txt"}, } w, err := NewWatcher(wCfg) assert.Nil(t, err, "wacher error") wt := w.(*watcher) - filePath := tc.file - file, err := os.Open(filePath) - if err != nil { - t.Fatal(err) - } - - fileInfo, err := file.Stat() - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, tc.expectIgnore, wt.ignoreFile(filePath, fileInfo)) + assert.Equal(t, tc.expectIgnore, wt.ignoreFile(tc.file)) }) } } @@ -259,3 +259,12 @@ func TestWatcherResolvePaths(t *testing.T) { }) } } + +func modifyFile(t *testing.T, filename string) { + f, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + assert.NoError(t, err) + + defer f.Close() + _, err = f.WriteString("\n") + assert.NoError(t, err) +} diff --git a/testdata/.hidden-file b/testdata/.hidden-file index e69de29..6f6f000 100644 --- a/testdata/.hidden-file +++ b/testdata/.hidden-file @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/testdata/mocks.go b/testdata/mocks.go index 36ddfb0..27f32d8 100644 --- a/testdata/mocks.go +++ b/testdata/mocks.go @@ -3,6 +3,7 @@ package testdata import ( "os/exec" + "github.com/fsnotify/fsnotify" "github.com/stretchr/testify/mock" ) @@ -78,9 +79,9 @@ type MockWacther struct { func (m *MockWacther) Watch() {} // Events ... -func (m *MockWacther) Events() chan string { +func (m *MockWacther) Events() chan []fsnotify.Event { args := m.Called() - return args.Get(0).(chan string) + return args.Get(0).(chan []fsnotify.Event) } // Errors ... diff --git a/testdata/server/main.go b/testdata/server/main.go index 71d1be9..0d02f41 100644 --- a/testdata/server/main.go +++ b/testdata/server/main.go @@ -21,3 +21,19 @@ func main() { log.Println("Starting server: Version", Version) log.Fatal(http.ListenAndServe(":8080", nil)) } + + + + + + + + + + + + + + + + diff --git a/testdata/server/main_test.go b/testdata/server/main_test.go index 6225fa1..fdb2a08 100644 --- a/testdata/server/main_test.go +++ b/testdata/server/main_test.go @@ -1,3 +1,19 @@ package main // an empty test file just to check during tests we can ignore _test.go files + + + + + + + + + + + + + + + + diff --git a/testdata/.hidden-folder/.gitkeep b/testdata/server/server.log similarity index 100% rename from testdata/.hidden-folder/.gitkeep rename to testdata/server/server.log diff --git a/watcher.go b/watcher.go deleted file mode 100644 index 5ca3a8c..0000000 --- a/watcher.go +++ /dev/null @@ -1,257 +0,0 @@ -package gaper - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - "time" - - zglob "github.com/mattn/go-zglob" -) - -// Watcher is a interface for the watch process -type Watcher interface { - Watch() - Errors() chan error - Events() chan string -} - -// watcher is a interface for the watch process -type watcher struct { - defaultIgnore bool - pollInterval int - watchItems map[string]bool - ignoreItems map[string]bool - allowedExtensions map[string]bool - events chan string - errors chan error -} - -// WatcherConfig defines the settings available for the watcher -type WatcherConfig struct { - DefaultIgnore bool - PollInterval int - WatchItems []string - IgnoreItems []string - Extensions []string -} - -// NewWatcher creates a new watcher -func NewWatcher(cfg WatcherConfig) (Watcher, error) { - if cfg.PollInterval == 0 { - cfg.PollInterval = DefaultPoolInterval - } - - if len(cfg.Extensions) == 0 { - cfg.Extensions = DefaultExtensions - } - - allowedExts := make(map[string]bool) - for _, ext := range cfg.Extensions { - allowedExts["."+ext] = true - } - - watchPaths, err := resolvePaths(cfg.WatchItems, allowedExts) - if err != nil { - return nil, err - } - - ignorePaths, err := resolvePaths(cfg.IgnoreItems, allowedExts) - if err != nil { - return nil, err - } - - logger.Debugf("Resolved watch paths: %v", watchPaths) - logger.Debugf("Resolved ignore paths: %v", ignorePaths) - return &watcher{ - events: make(chan string), - errors: make(chan error), - defaultIgnore: cfg.DefaultIgnore, - pollInterval: cfg.PollInterval, - watchItems: watchPaths, - ignoreItems: ignorePaths, - allowedExtensions: allowedExts, - }, nil -} - -var startTime = time.Now() -var errDetectedChange = errors.New("done") - -// Watch starts watching for file changes -func (w *watcher) Watch() { - for { - for watchPath := range w.watchItems { - fileChanged, err := w.scanChange(watchPath) - if err != nil { - w.errors <- err - return - } - - if fileChanged != "" { - w.events <- fileChanged - startTime = time.Now() - } - } - - time.Sleep(time.Duration(w.pollInterval) * time.Millisecond) - } -} - -// Events get events occurred during the watching -// these events are emitted only a file changing is detected -func (w *watcher) Events() chan string { - return w.events -} - -// Errors get errors occurred during the watching -func (w *watcher) Errors() chan error { - return w.errors -} - -func (w *watcher) scanChange(watchPath string) (string, error) { - logger.Debug("Watching ", watchPath) - - var fileChanged string - - err := filepath.Walk(watchPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - // Ignore attempt to acess go temporary unmask - if strings.Contains(err.Error(), "-go-tmp-umask") { - return filepath.SkipDir - } - - return fmt.Errorf("couldn't walk to path \"%s\": %v", path, err) - } - - if w.ignoreFile(path, info) { - return skipFile(info) - } - - ext := filepath.Ext(path) - if _, ok := w.allowedExtensions[ext]; ok && info.ModTime().After(startTime) { - fileChanged = path - return errDetectedChange - } - - return nil - }) - - if err != nil && err != errDetectedChange { - return "", err - } - - return fileChanged, nil -} - -func (w *watcher) ignoreFile(path string, info os.FileInfo) bool { - // if a file has been deleted after gaper was watching it - // info will be nil in the other iterations - if info == nil { - return true - } - - // check if preset ignore is enabled - if w.defaultIgnore { - // check for hidden files and directories - if name := info.Name(); name[0] == '.' && name != "." { - return true - } - - // check if it is a Go testing file - if strings.HasSuffix(path, "_test.go") { - return true - } - - // check if it is the vendor folder - if info.IsDir() && info.Name() == "vendor" { - return true - } - } - - if _, ignored := w.ignoreItems[path]; ignored { - return true - } - - return false -} - -func resolvePaths(paths []string, extensions map[string]bool) (map[string]bool, error) { - result := map[string]bool{} - - for _, path := range paths { - matches := []string{path} - - isGlob := strings.Contains(path, "*") - if isGlob { - var err error - matches, err = zglob.Glob(path) - if err != nil { - return nil, fmt.Errorf("couldn't resolve glob path \"%s\": %v", path, err) - } - } - - for _, match := range matches { - // ignore existing files that don't match the allowed extensions - if f, err := os.Stat(match); !os.IsNotExist(err) && !f.IsDir() { - if ext := filepath.Ext(match); ext != "" { - if _, ok := extensions[ext]; !ok { - continue - } - } - } - - if _, ok := result[match]; !ok { - result[match] = true - } - } - } - - removeOverlappedPaths(result) - - return result, nil -} - -// remove overlapped paths so it makes the scan for changes later faster and simpler -func removeOverlappedPaths(mapPaths map[string]bool) { - startDot := regexp.MustCompile(`^\./`) - - for p1 := range mapPaths { - p1 = startDot.ReplaceAllString(p1, "") - - // skip to next item if this path has already been checked - if v, ok := mapPaths[p1]; ok && !v { - continue - } - - for p2 := range mapPaths { - p2 = startDot.ReplaceAllString(p2, "") - - if p1 == p2 { - continue - } - - if strings.HasPrefix(p2, p1) { - mapPaths[p2] = false - } else if strings.HasPrefix(p1, p2) { - mapPaths[p1] = false - } - } - } - - // cleanup path list - for p := range mapPaths { - if !mapPaths[p] { - delete(mapPaths, p) - } - } -} - -func skipFile(info os.FileInfo) error { - if info.IsDir() { - return filepath.SkipDir - } - return nil -}