Build and Historic build updates

Modified GOPATH to not modify build with go.mod
Updated go.mod to version 1.12
Updated harness to setup listener before killing process
Updated notvendored flag to --no-vendor
Updated command_config to ensure no-vendor can be build
Added additional checks in source path lookup

.travis.yml

@@ -28,10 +28,11 @@ install:
   - 'if [[ "$TRAVIS_BRANCH" == "master" ]]; then export REVEL_BRANCH="master"; fi'
   - 'echo "Travis branch: $TRAVIS_BRANCH, Revel dependency branch: $REVEL_BRANCH"'
   # Since travis already checks out go build the commandline tool (revel)
-  - go get -v github.com/revel/cmd/revel
+  - mkdir $HOME/GOPATH_PROTECTED
-  - echo $GOPATH
+  - export GOPATH=$HOME/GOPATH_PROTECTED
-  - echo $PATH
+  - go build -o $HOME/gopath/bin/revel github.com/revel/cmd/revel
   - pwd
+  - env
 script:
   - go test -v github.com/revel/cmd/revel/...
 
@@ -54,6 +55,12 @@ script:
   - revel package --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v
   - revel package --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v -m prod
 
+  # Check build works with no-vendor flag
+  - cd $GOPATH
+  - export GO111MODULE=auto
+  - revel new  -a my/testapp2 --no-vendor
+  - revel test -a my/testapp2
+
 matrix:
   allow_failures:
     - go: tip

go.mod

@@ -1,6 +1,6 @@
 module github.com/revel/cmd
 
-go 1.13
+go 1.12
 
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
@@ -8,7 +8,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 // indirect
 	github.com/jessevdk/go-flags v1.4.0
-	github.com/mattn/go-colorable v0.1.4
+	github.com/mattn/go-colorable v0.1.6
 	github.com/myesui/uuid v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/revel/config v0.21.0
@@ -19,7 +19,8 @@ require (
 	github.com/stretchr/testify v1.4.0
 	github.com/twinj/uuid v1.0.0 // indirect
 	github.com/xeonx/timeago v1.0.0-rc4 // indirect
-	golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect
+	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 // indirect
+	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect
 	golang.org/x/tools v0.0.0-20200219054238-753a1d49df85
 	gopkg.in/fsnotify/fsnotify.v1 v1.4.7
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22

harness/app.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"time"
+	"sync"
 
 	"github.com/revel/cmd/model"
 	"github.com/revel/cmd/utils"
@@ -64,8 +65,8 @@ func NewAppCmd(binPath string, port int, runMode string, paths *model.RevelConta
 func (cmd AppCmd) Start(c *model.CommandConfig) error {
 	listeningWriter := &startupListeningWriter{os.Stdout, make(chan bool), c, &bytes.Buffer{}}
 	cmd.Stdout = listeningWriter
-	utils.Logger.Info("Exec app:", "path", cmd.Path, "args", cmd.Args, "dir", cmd.Dir, "env", cmd.Env)
 	utils.CmdInit(cmd.Cmd, !c.Vendored, c.AppPath)
+	utils.Logger.Info("Exec app:", "path", cmd.Path, "args", cmd.Args, "dir", cmd.Dir, "env", cmd.Env)
 	if err := cmd.Cmd.Start(); err != nil {
 		utils.Logger.Fatal("Error running:", "error", err)
 	}
@@ -107,10 +108,30 @@ func (cmd AppCmd) Kill() {
 		// server before this can, this check will ensure the process is still running
 		if _, err := os.FindProcess(int(cmd.Process.Pid));err!=nil {
 			// Server has already exited
-			utils.Logger.Info("Killing revel server pid", "pid", cmd.Process.Pid)
+			utils.Logger.Info("Server not running revel server pid", "pid", cmd.Process.Pid)
 			return
 		}
 
+		// Wait for the shutdown channel
+		waitMutex := &sync.WaitGroup{}
+		waitMutex.Add(1)
+		ch := make(chan bool, 1)
+		go func() {
+			waitMutex.Done()
+			s, err := cmd.Process.Wait()
+			defer func() {
+				ch <- true
+			}()
+			if err != nil {
+				utils.Logger.Info("Wait failed for process ", "error", err)
+			}
+			if s != nil {
+				utils.Logger.Info("Revel App exited", "state", s.String())
+			}
+		}()
+		// Wait for the channel to begin waiting
+		waitMutex.Wait()
+
 		// Send an interrupt signal to allow for a graceful shutdown
 		utils.Logger.Info("Killing revel server pid", "pid", cmd.Process.Pid)
 		var err error
@@ -129,20 +150,6 @@ func (cmd AppCmd) Kill() {
 			return
 		}
 
-		// Wait for the shutdown
-		ch := make(chan bool, 1)
-		go func() {
-			s, err := cmd.Process.Wait()
-			defer func() {
-				ch <- true
-			}()
-			if err != nil {
-				utils.Logger.Info("Wait failed for process ", "error", err)
-			}
-			if s != nil {
-				utils.Logger.Info("Revel App exited", "state", s.String())
-			}
-		}()
 
 		// Use a timer to ensure that the process exits
 		utils.Logger.Info("Waiting to exit")

harness/build.go

@@ -102,28 +102,8 @@ func Build(c *model.CommandConfig, paths *model.RevelContainer) (_ *App, err err
 		utils.Logger.Fatal("Go executable not found in PATH.")
 	}
 
-	// Detect if deps tool should be used (is there a vendor folder ?)
+	// Binary path is a combination of target/app directory, app's import path and its name.
-	useVendor := utils.DirExists(filepath.Join(paths.BasePath, "vendor"))
+	binName := filepath.Join("target", "app", paths.ImportPath, filepath.Base(paths.BasePath))
-	basePath := paths.BasePath
-	for !useVendor {
-		basePath = filepath.Dir(basePath)
-		found := false
-		// Check to see if we are still in the GOPATH
-		for _, gopath := range filepath.SplitList(build.Default.GOPATH) {
-			if strings.HasPrefix(basePath, gopath) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			break
-		} else {
-			useVendor = utils.DirExists(filepath.Join(basePath, "vendor"))
-		}
-	}
-
-	// Binary path is a combination of BasePath/target/app directory, app's import path and its name.
-	binName := filepath.Join(paths.BasePath, "target", "app", paths.ImportPath, filepath.Base(paths.BasePath))
 
 	// Change binary path for Windows build
 	goos := runtime.GOOS

model/command/new.go

@@ -6,7 +6,7 @@ type (
 		ImportCommand
 		SkeletonPath string `short:"s" long:"skeleton" description:"Path to skeleton folder (Must exist on GO PATH)" required:"false"`
 		Package      string `short:"p" long:"package" description:"The package name, this becomes the repfix to the app name, if defined vendored is set to true" required:"false"`
-		NotVendored  bool   `long:"not-vendor" description:"True if project should not be configured with a go.mod, this requires you to have the project on the GOPATH"`
+		NotVendored  bool   `long:"no-vendor" description:"True if project should not be configured with a go.mod, this requires you to have the project on the GOPATH, this is only compatible with go versions v1.12 or older"`
 		Run          bool   `short:"r" long:"run" description:"True if you want to run the application right away"`
 		Callback func() error
 	}

model/command_config.go

@@ -41,7 +41,7 @@ type (
 		ImportPath        string                                                                                                                      // The import path (relative to a GOPATH)
 		GoPath            string                                                                                                                      // The GoPath
 		GoCmd             string                                                                                                                      // The full path to the go executable
-		SrcRoot           string                                                                                                                      // The source root
+	                                                                                                                                                //SrcRoot           string                                                                                                                      // The source root
 		AppPath           string                                                                                                                      // The application path (absolute)
 		AppName           string                                                                                                                      // The application name
 		HistoricBuildMode bool     `long:"historic-build-mode" description:"If set the code is scanned using the original parsers, not the go.1.11+"` // True if debug is active
@@ -143,7 +143,7 @@ func (c *CommandConfig) UpdateImportPath() error {
 }
 
 func (c *CommandConfig) initAppFolder() (err error) {
-	utils.Logger.Info("initAppFolder", "vendored", c.Vendored)
+	utils.Logger.Info("initAppFolder", "vendored", c.Vendored, "build-gopath", build.Default.GOPATH, "gopath-env", os.Getenv("GOPATH"))
 
 	// check for go executable
 	c.GoCmd, err = exec.LookPath("go")
@@ -184,30 +184,17 @@ func (c *CommandConfig) initAppFolder() (err error) {
 			if strings.Index(line, "module ") == 0 {
 				c.ImportPath = strings.TrimSpace(strings.Split(line, "module")[1])
 				c.AppPath = appFolder
-				c.SrcRoot = appFolder
+				//c.SrcRoot = appFolder
-				utils.Logger.Info("Set application path and package based on go mod", "path", c.AppPath, "sourceroot", c.SrcRoot)
+				utils.Logger.Info("Set application path and package based on go mod", "path", c.AppPath)
 				return nil
 			}
 		}
-	}
+		// c.SrcRoot = appFolder
-
+		c.AppPath = appFolder
-	utils.Logger.Debug("Trying to set path based on gopath")
+	} else if c.Index != NEW || (c.Index == NEW && c.New.NotVendored) {
-	// lookup go path
+		workingDir, _ := os.Getwd()
-	c.GoPath = build.Default.GOPATH
+		goPathList := filepath.SplitList(c.GoPath)
-	if c.GoPath == "" {
+		bestpath := ""
-		utils.Logger.Fatal("Abort: GOPATH environment variable is not set. " +
-			"Please refer to http://golang.org/doc/code.html to configure your Go environment.")
-	}
-
-	// revel/revel#1004 choose go path relative to current working directory
-
-	// What we want to do is to add the import to the end of the
-	// gopath, and discover which import exists - If none exist this is an error except in the case
-	// where we are dealing with new which is a special case where we will attempt to target the working directory first
-	workingDir, _ := os.Getwd()
-	goPathList := filepath.SplitList(c.GoPath)
-	bestpath := ""
-	if !c.Vendored {
 		for _, path := range goPathList {
 			if c.Index == NEW {
 				// If the GOPATH is part of the working dir this is the most likely target
@@ -216,61 +203,48 @@ func (c *CommandConfig) initAppFolder() (err error) {
 				}
 			} else {
 				if utils.Exists(filepath.Join(path, "src", c.ImportPath)) {
-					c.SrcRoot = path
+					bestpath = path
 					break
 				}
 			}
 		}
-		if len(c.SrcRoot) == 0 && len(bestpath) > 0 {
+
-			c.SrcRoot = bestpath
+		utils.Logger.Info("Source root", "cwd", workingDir, "gopath", c.GoPath, "c.ImportPath", c.ImportPath, "bestpath", bestpath)
+		if len(bestpath) > 0 {
+			c.AppPath = filepath.Join(bestpath, "src", c.ImportPath)
 		}
+		// Recalculate the appFolder because we are using a GOPATH
 
 	} else {
-		c.SrcRoot = appFolder
+		// This is new and not vendored, so the app path is the appFolder
+		c.AppPath = appFolder
 	}
 
-	utils.Logger.Info("Source root", "path", c.SrcRoot, "cwd", workingDir, "gopath", c.GoPath, "bestpath", bestpath)
-
-	// If source root is empty and this isn't a version then skip it
-	if len(c.SrcRoot) == 0 {
-		if c.Index == NEW {
-			c.SrcRoot = c.New.ImportPath
-		} else {
-			if c.Index != VERSION {
-				utils.Logger.Fatal("Abort: could not create a Revel application outside of GOPATH.")
-			}
-			return nil
-		}
-	}
-
-	// set go src path
-	if c.Vendored {
-		c.AppPath = c.SrcRoot
-
-	} else {
-		c.SrcRoot = filepath.Join(c.SrcRoot, "src")
-
-		c.AppPath = filepath.Join(c.SrcRoot, filepath.FromSlash(c.ImportPath))
-	}
 	utils.Logger.Info("Set application path", "path", c.AppPath)
 	return nil
 }
 
 // Used to initialize the package resolver
 func (c *CommandConfig) InitPackageResolver() {
+	c.initGoPaths()
 	utils.Logger.Info("InitPackageResolver", "useVendor", c.Vendored, "path", c.AppPath)
 
 	// This should get called when needed
 	c.PackageResolver = func(pkgName string) error {
-		//useVendor := utils.DirExists(filepath.Join(c.AppPath, "vendor"))
-
-		//var getCmd *exec.Cmd
 		utils.Logger.Info("Request for package ", "package", pkgName, "use vendor", c.Vendored)
+		var getCmd *exec.Cmd
 		if c.Vendored {
-			goModCmd := exec.Command("go", "mod", "tidy")
+			getCmd = exec.Command(c.GoCmd, "mod", "tidy")
-			utils.CmdInit(goModCmd, !c.Vendored, c.AppPath)
+		} else {
-			goModCmd.Run()
+			utils.Logger.Info("No vendor folder detected, not using dependency manager to import package", "package", pkgName)
-			return nil
+			getCmd = exec.Command(c.GoCmd, "get", "-u", pkgName)
+		}
+
+		utils.CmdInit(getCmd, !c.Vendored, c.AppPath)
+		utils.Logger.Info("Go get command ", "exec", getCmd.Path, "dir", getCmd.Dir, "args", getCmd.Args, "env", getCmd.Env, "package", pkgName)
+		output, err := getCmd.CombinedOutput()
+		if err != nil {
+			utils.Logger.Error("Failed to import package", "error", err, "gopath", build.Default.GOPATH, "GO-ROOT", build.Default.GOROOT, "output", string(output))
 		}
 
 		return nil
@@ -278,15 +252,8 @@ func (c *CommandConfig) InitPackageResolver() {
 }
 
 // lookup and set Go related variables
-func (c *CommandConfig) InitGoPathsOld() {
+func (c *CommandConfig) initGoPaths() {
-	utils.Logger.Info("InitGoPaths")
+	utils.Logger.Info("InitGoPaths", "vendored", c.Vendored)
-	// lookup go path
-	c.GoPath = build.Default.GOPATH
-	if c.GoPath == "" {
-		utils.Logger.Fatal("Abort: GOPATH environment variable is not set. " +
-			"Please refer to http://golang.org/doc/code.html to configure your Go environment.")
-	}
-
 	// check for go executable
 	var err error
 	c.GoCmd, err = exec.LookPath("go")
@@ -294,50 +261,46 @@ func (c *CommandConfig) InitGoPathsOld() {
 		utils.Logger.Fatal("Go executable not found in PATH.")
 	}
 
+	if c.Vendored {
+		return
+	}
+
+	// lookup go path
+	c.GoPath = build.Default.GOPATH
+	if c.GoPath == "" {
+		utils.Logger.Fatal("Abort: GOPATH environment variable is not set. " +
+			"Please refer to http://golang.org/doc/code.html to configure your Go environment.")
+	}
+	return
+	//todo determine if the rest needs to happen
+
+
 	// revel/revel#1004 choose go path relative to current working directory
 
 	// What we want to do is to add the import to the end of the
 	// gopath, and discover which import exists - If none exist this is an error except in the case
 	// where we are dealing with new which is a special case where we will attempt to target the working directory first
-	workingDir, _ := os.Getwd()
+	/*
-	goPathList := filepath.SplitList(c.GoPath)
+		// If source root is empty and this isn't a version then skip it
-	bestpath := ""
+		if len(c.SrcRoot) == 0 {
-	for _, path := range goPathList {
+			if c.Index == NEW {
-		if c.Index == NEW {
+				c.SrcRoot = c.New.ImportPath
-			// If the GOPATH is part of the working dir this is the most likely target
+			} else {
-			if strings.HasPrefix(workingDir, path) {
+				if c.Index != VERSION {
-				bestpath = path
+					utils.Logger.Fatal("Abort: could not create a Revel application outside of GOPATH.")
-			}
+				}
-		} else {
+				return
-			if utils.Exists(filepath.Join(path, "src", c.ImportPath)) {
-				c.SrcRoot = path
-				break
 			}
 		}
-	}
 
-	utils.Logger.Info("Source root", "path", c.SrcRoot, "cwd", workingDir, "gopath", c.GoPath, "bestpath", bestpath)
+		// set go src path
-	if len(c.SrcRoot) == 0 && len(bestpath) > 0 {
+		c.SrcRoot = filepath.Join(c.SrcRoot, "src")
-		c.SrcRoot = bestpath
-	}
 
-	// If source root is empty and this isn't a version then skip it
+		c.AppPath = filepath.Join(c.SrcRoot, filepath.FromSlash(c.ImportPath))
-	if len(c.SrcRoot) == 0 {
+		utils.Logger.Info("Set application path", "path", c.AppPath)
-		if c.Index == NEW {
-			c.SrcRoot = c.New.ImportPath
-		} else {
-			if c.Index != VERSION {
-				utils.Logger.Fatal("Abort: could not create a Revel application outside of GOPATH.")
-			}
-			return
-		}
-	}
 
-	// set go src path
+	 */
-	c.SrcRoot = filepath.Join(c.SrcRoot, "src")
 
-	c.AppPath = filepath.Join(c.SrcRoot, filepath.FromSlash(c.ImportPath))
-	utils.Logger.Info("Set application path", "path", c.AppPath)
 }
 
 // Sets the versions on the command config

revel/build.go

@@ -59,6 +59,7 @@ func updateBuildConfig(c *model.CommandConfig, args []string) bool {
 
 // The main entry point to build application from command line
 func buildApp(c *model.CommandConfig) (err error) {
+
 	appImportPath, destPath, mode := c.ImportPath, c.Build.TargetPath, DefaultRunMode
 	if len(c.Build.Mode) > 0 {
 		mode = c.Build.Mode
@@ -113,7 +114,7 @@ func buildCopyFiles(c *model.CommandConfig, app *harness.App, revel_paths *model
 	srcPath := filepath.Join(destPath, "src")
 	destBinaryPath := filepath.Join(destPath, filepath.Base(app.BinaryPath))
 	tmpRevelPath := filepath.Join(srcPath, filepath.FromSlash(model.RevelImportPath))
-	if err = utils.CopyFile(destBinaryPath, app.BinaryPath); err != nil {
+	if err = utils.CopyFile(destBinaryPath, filepath.Join(revel_paths.BasePath, app.BinaryPath)); err != nil {
 		return
 	}
 	utils.MustChmod(destBinaryPath, 0755)

revel/new_test.go

@@ -18,6 +18,12 @@ func TestNew(t *testing.T) {
 		c := newApp("new-test", model.NEW, nil, a)
 		a.Nil(main.Commands[model.NEW].RunWith(c), "New failed")
 	})
+	t.Run("New-NotVendoredmode", func(t *testing.T) {
+		a := assert.New(t)
+		c := newApp("new-notvendored", model.NEW, nil, a)
+		c.New.NotVendored = true
+		a.Nil(main.Commands[model.NEW].RunWith(c), "New failed")
+	})
 	t.Run("Path", func(t *testing.T) {
 		a := assert.New(t)
 		c := newApp("new/test/a", model.NEW, nil, a)

revel/revel.go

@@ -91,6 +91,9 @@ func main() {
 		utils.InitLogger(wd, logger.LvlWarn)
 	}
 
+	// Setup package resolver
+	c.InitPackageResolver()
+
 	if err := c.UpdateImportPath(); err != nil {
 		utils.Logger.Error(err.Error())
 		parser.WriteHelp(os.Stdout)
@@ -100,12 +103,6 @@ func main() {
 	command := Commands[c.Index]
 	println("Revel executing:", command.Short)
 
-	// Setting go paths
-	// c.InitGoPaths()
-
-	// Setup package resolver
-	c.InitPackageResolver()
-
 	if err := command.RunWith(c); err != nil {
 		utils.Logger.Error("Unable to execute", "error", err)
 		os.Exit(1)

utils/file.go

@@ -322,7 +322,8 @@ func Empty(dirname string) bool {
 // Find the full source dir for the import path, uses the build.Default.GOPATH to search for the directory
 func FindSrcPaths(appPath string, packageList []string, packageResolver func(pkgName string) error) (sourcePathsmap map[string]string, err error) {
 	sourcePathsmap, missingList, err := findSrcPaths(appPath, packageList)
-	if err != nil && packageResolver != nil {
+	if err != nil && packageResolver != nil || len(missingList)>0 {
+		Logger.Info("Failed to find package, attempting to call resolver for missing packages","missing packages",missingList)
 		for _, item := range missingList {
 			if err = packageResolver(item); err != nil {
 				return
@@ -351,22 +352,28 @@ func findSrcPaths(appPath string, packagesList []string) (sourcePathsmap map[str
 		Dir:appPath,
 	}
 	sourcePathsmap = map[string]string{}
+	Logger.Infof("Environment path %s root %s config env %s", os.Getenv("GOPATH"), os.Getenv("GOROOT"),config.Env)
 
 	pkgs, err := packages.Load(config, packagesList...)
+	Logger.Infof("Environment path %s root %s config env %s", os.Getenv("GOPATH"), os.Getenv("GOROOT"),config.Env)
 	Logger.Info("Loaded packages ", "len results", len(pkgs), "error", err, "basedir", appPath)
 	for _, packageName := range packagesList {
-		found := false
+		 found := false
 		log := Logger.New("seeking", packageName)
 		for _, pck := range pkgs {
 			log.Info("Found package", "package", pck.ID)
 			if pck.ID == packageName {
 				if pck.Errors != nil && len(pck.Errors) > 0 {
 					log.Info("Error ", "count", len(pck.Errors), "App Import Path", pck.ID, "errors", pck.Errors)
+					continue
+
 				}
 				//a,_ := pck.MarshalJSON()
 				log.Info("Found ", "count", len(pck.GoFiles), "App Import Path", pck.ID, "apppath", appPath)
-				sourcePathsmap[packageName] = filepath.Dir(pck.GoFiles[0])
+				if len(pck.GoFiles)>0 {
-				found = true
+					sourcePathsmap[packageName] = filepath.Dir(pck.GoFiles[0])
+					found = true
+				}
 			}
 		}
 		if !found {
@@ -378,5 +385,6 @@ func findSrcPaths(appPath string, packagesList []string) (sourcePathsmap map[str
 			missingList = append(missingList, packageName)
 		}
 	}
+
 	return
 }