diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a6e1c32 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module github.com/revel/cmd + +go 1.13 + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/agtorre/gocolorize v1.0.0 + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 // indirect + github.com/jessevdk/go-flags v1.4.0 + github.com/mattn/go-colorable v0.1.4 + github.com/myesui/uuid v1.0.0 // indirect + github.com/pkg/errors v0.9.1 + github.com/revel/config v0.21.0 + github.com/revel/log15 v2.11.20+incompatible + github.com/revel/pathtree v0.0.0-20140121041023-41257a1839e9 // indirect + github.com/revel/revel v0.21.0 + github.com/stretchr/testify v1.4.0 + github.com/twinj/uuid v1.0.0 // indirect + github.com/xeonx/timeago v1.0.0-rc4 // indirect + golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect + golang.org/x/tools v0.0.0-20200219054238-753a1d49df85 + gopkg.in/fsnotify/fsnotify.v1 v1.4.7 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/stack.v0 v0.0.0-20141108040640-9b43fcefddd0 + gopkg.in/stretchr/testify.v1 v1.2.2 // indirect +) diff --git a/harness/build.go b/harness/build.go index ea6b7ef..b8bc7f9 100644 --- a/harness/build.go +++ b/harness/build.go @@ -19,8 +19,10 @@ import ( "time" "github.com/revel/cmd/model" - "github.com/revel/cmd/parser" + _ "github.com/revel/cmd/parser" "github.com/revel/cmd/utils" + "github.com/revel/cmd/parser2" + "github.com/revel/cmd/parser" ) var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"") @@ -40,7 +42,13 @@ func Build(c *model.CommandConfig, paths *model.RevelContainer) (_ *App, err err // First, clear the generated files (to avoid them messing with ProcessSource). cleanSource(paths, "tmp", "routes") - sourceInfo, err := parser.ProcessSource(paths) + var sourceInfo *model.SourceInfo + + if c.HistoricBuildMode { + sourceInfo, err = parser.ProcessSource(paths) + } else { + sourceInfo, err = parser2.ProcessSource(paths) + } if err != nil { return } @@ -151,7 +159,6 @@ func Build(c *model.CommandConfig, paths *model.RevelContainer) (_ *App, err err if len(c.BuildFlags) == 0 { flags = []string{ "build", - "-i", "-ldflags", versionLinkerFlags, "-tags", buildTags, "-o", binName} @@ -178,7 +185,7 @@ func Build(c *model.CommandConfig, paths *model.RevelContainer) (_ *App, err err // This is Go main path gopath := c.GoPath for _, o := range paths.ModulePathMap { - gopath += string(filepath.ListSeparator) + o + gopath += string(filepath.ListSeparator) + o.Path } // Note: It's not applicable for filepath.* usage diff --git a/model/command_config.go b/model/command_config.go index 93b355b..6cbbe76 100644 --- a/model/command_config.go +++ b/model/command_config.go @@ -2,6 +2,9 @@ package model import ( "fmt" + "github.com/revel/cmd" +// "github.com/revel/cmd/logger" + "github.com/revel/cmd/utils" "go/ast" "go/build" "go/parser" @@ -11,10 +14,7 @@ import ( "os/exec" "path/filepath" "strings" - - "github.com/revel/cmd" - "github.com/revel/cmd/logger" - "github.com/revel/cmd/utils" + "runtime" ) // The constants @@ -45,15 +45,17 @@ type ( SrcRoot string // The source root AppPath string // The application path (absolute) AppName string // The application name - Vendored bool // True if the application is vendored + HistoricBuildMode bool `long:"historic-build-mode" description:"If set the code is scanned using the original parsers, not the go.1.11+"` // True if debug is active + Vendored bool // True if the application is vendored PackageResolver func(pkgName string) error // a packge resolver for the config BuildFlags []string `short:"X" long:"build-flags" description:"These flags will be used when building the application. May be specified multiple times, only applicable for Build, Run, Package, Test commands"` // The new command New struct { - ImportPath string `short:"a" long:"application-path" description:"Path to application folder" required:"false"` - SkeletonPath string `short:"s" long:"skeleton" description:"Path to skeleton folder (Must exist on GO PATH)" required:"false"` - Vendored bool `short:"V" long:"vendor" description:"True if project should contain a vendor folder to be initialized. Creates the vendor folder and the 'Gopkg.toml' file in the root"` - Run bool `short:"r" long:"run" description:"True if you want to run the application right away"` + ImportPath string `short:"a" long:"application-path" description:"Path to application folder" required:"false"` + SkeletonPath string `short:"s" long:"skeleton" description:"Path to skeleton folder (Must exist on GO PATH)" required:"false"` + Package string `short:"p" long:"package" description:"The package name, this becomes the repfix to the app name, if defined vendored is set to true" required:"false"` + Vendored bool `short:"V" long:"vendor" description:"True if project should contain a vendor folder to be initialized. Creates the vendor folder and the 'Gopkg.toml' file in the root"` + Run bool `short:"r" long:"run" description:"True if you want to run the application right away"` } `command:"new"` // The build command Build struct { @@ -89,7 +91,7 @@ type ( // The version command Version struct { ImportPath string `short:"a" long:"application-path" description:"Path to application folder" required:"false"` - Update bool `short:"u" long:"update" description:"Update the framework and modules" required:"false"` + Update bool `short:"u" long:"Update the framework and modules" required:"false"` } `command:"version"` } ) @@ -103,14 +105,19 @@ func (c *CommandConfig) UpdateImportPath() error { importPath = c.New.ImportPath case RUN: importPath = c.Run.ImportPath + c.Vendored = utils.Exists(filepath.Join(importPath,"src","go.mod")) case BUILD: importPath = c.Build.ImportPath + c.Vendored = utils.Exists(filepath.Join(importPath,"src","go.mod")) case PACKAGE: importPath = c.Package.ImportPath + c.Vendored = utils.Exists(filepath.Join(importPath,"src","go.mod")) case CLEAN: importPath = c.Clean.ImportPath + c.Vendored = utils.Exists(filepath.Join(importPath,"src","go.mod")) case TEST: importPath = c.Test.ImportPath + c.Vendored = utils.Exists(filepath.Join(importPath,"src","go.mod")) case VERSION: importPath = c.Version.ImportPath required = false @@ -135,8 +142,7 @@ func (c *CommandConfig) UpdateImportPath() error { if strings.HasPrefix(currentPath, path) && len(currentPath) > len(path)+1 { importPath = currentPath[len(path)+1:] // Remove the source from the path if it is there - isSRC := strings.ToLower(importPath[0:4]) - if len(importPath) > 4 && (isSRC == "src/" || isSRC == "src\\") { + if len(importPath) > 4 && strings.ToLower(importPath[0:4]) == "src/" { importPath = importPath[4:] } else if importPath == "src" { if c.Index != VERSION { @@ -151,12 +157,14 @@ func (c *CommandConfig) UpdateImportPath() error { } c.ImportPath = importPath - utils.Logger.Info("Returned import path", "path", importPath, "buildpath", build.Default.GOPATH) + // We need the source root determined at this point to check the setversions + c.initAppFolder() + utils.Logger.Info("Returned import path", "path", importPath) if required && c.Index != NEW { if err := c.SetVersions(); err != nil { utils.Logger.Panic("Failed to fetch revel versions", "error", err) } - if err := c.FrameworkVersion.CompatibleFramework(c); err != nil { + if err:=c.FrameworkVersion.CompatibleFramework(c);err!=nil { utils.Logger.Fatal("Compatibility Error", "message", err, "Revel framework version", c.FrameworkVersion.String(), "Revel tool version", c.CommandVersion.String()) } @@ -165,34 +173,130 @@ func (c *CommandConfig) UpdateImportPath() error { if !required { return nil } - if len(importPath) == 0 { + if len(importPath) == 0 { return fmt.Errorf("Unable to determine import path from : %s", importPath) } return nil } +func (c *CommandConfig) initAppFolder() (err error) { + utils.Logger.Info("initAppFolder") + // check for go executable + c.GoCmd, err = exec.LookPath("go") + if err != nil { + utils.Logger.Fatal("Go executable not found in PATH.") + } + + // First try to determine where the application is located - this should be the import value + appFolder := c.ImportPath + wd,err := os.Getwd() + if len(appFolder) == 0 { + // We will assume the working directory is the appFolder + appFolder = wd + } else if strings.LastIndex(wd,appFolder)==len(wd)-len(appFolder) { + // Check for existence of an /app folder + if utils.Exists(filepath.Join(wd,"app")) { + appFolder = wd + } else { + appFolder = filepath.Join(wd,appFolder) + } + } else if strings.Contains(appFolder,".") { + appFolder = filepath.Join(wd,filepath.Base(c.ImportPath)) + } else if !filepath.IsAbs(appFolder) { + appFolder = filepath.Join(wd,appFolder) + } + + utils.Logger.Info("Determined app folder to be", "folder",appFolder, "working",wd) + + // Use app folder to read the go.mod if it exists and extract the package information + goModFile := filepath.Join(appFolder,"go.mod") + if utils.Exists(goModFile) { + c.Vendored = true + file,err:=ioutil.ReadFile(goModFile) + if err!=nil { + return err + } + for _,line := range strings.Split(string(file),"\n") { + if strings.Index(line,"module ")==0 { + c.ImportPath = strings.TrimSpace(strings.Split(line,"module")[1]) + c.AppPath = appFolder + c.SrcRoot = appFolder + utils.Logger.Info("Set application path and package based on go mod", "path", c.AppPath, "sourceroot", c.SrcRoot) + return nil + } + } + } + + utils.Logger.Fatal("Trying to set path based on gopath") + // lookup go path + c.GoPath = build.Default.GOPATH + if c.GoPath == "" { + utils.Logger.Fatal("Abort: GOPATH environment variable is not set. " + + "Please refer to http://golang.org/doc/code.html to configure your Go environment.") + } + + // revel/revel#1004 choose go path relative to current working directory + + // What we want to do is to add the import to the end of the + // gopath, and discover which import exists - If none exist this is an error except in the case + // where we are dealing with new which is a special case where we will attempt to target the working directory first + workingDir, _ := os.Getwd() + goPathList := filepath.SplitList(c.GoPath) + bestpath := "" + for _, path := range goPathList { + if c.Index == NEW { + // If the GOPATH is part of the working dir this is the most likely target + if strings.HasPrefix(workingDir, path) { + bestpath = path + } + } else { + if utils.Exists(filepath.Join(path, "src", c.ImportPath)) { + c.SrcRoot = path + break + } + } + } + + utils.Logger.Info("Source root", "path", c.SrcRoot, "cwd", workingDir, "gopath", c.GoPath, "bestpath",bestpath) + if len(c.SrcRoot) == 0 && len(bestpath) > 0 { + c.SrcRoot = bestpath + } + + // If source root is empty and this isn't a version then skip it + if len(c.SrcRoot) == 0 { + if c.Index == NEW { + c.SrcRoot = c.New.ImportPath + } else { + if c.Index != VERSION { + utils.Logger.Fatal("Abort: could not create a Revel application outside of GOPATH.") + } + return nil + } + } + + // set go src path + c.SrcRoot = filepath.Join(c.SrcRoot, "src") + + c.AppPath = filepath.Join(c.SrcRoot, filepath.FromSlash(c.ImportPath)) + utils.Logger.Info("Set application path", "path", c.AppPath) + return nil +} + // Used to initialize the package resolver func (c *CommandConfig) InitPackageResolver() { - c.Vendored = utils.DirExists(filepath.Join(c.AppPath, "vendor")) + c.Vendored = utils.DirExists(filepath.Join(c.AppPath, "go.mod")) if c.Index == NEW && c.New.Vendored { c.Vendored = true } utils.Logger.Info("InitPackageResolver", "useVendor", c.Vendored, "path", c.AppPath) - var ( - depPath string - err error - ) if c.Vendored { - utils.Logger.Info("Vendor folder detected, scanning for deps in path") - depPath, err = exec.LookPath("dep") - if err != nil { + utils.Logger.Info("Vendor folder detected, for go version") + if runtime.Version()!="" { // Do not halt build unless a new package needs to be imported - utils.Logger.Fatal("Build: `dep` executable not found in PATH, but vendor folder detected." + - "Packages can only be added automatically to the vendor folder using the `dep` tool. " + - "You can install the `dep` tool by doing a `go get -u github.com/golang/dep/cmd/dep`") + utils.Logger.Fatal(`Go version 1.11 or newer is required to build`) } } @@ -200,45 +304,42 @@ func (c *CommandConfig) InitPackageResolver() { c.PackageResolver = func(pkgName string) error { //useVendor := utils.DirExists(filepath.Join(c.AppPath, "vendor")) - var getCmd *exec.Cmd + //var getCmd *exec.Cmd utils.Logger.Info("Request for package ", "package", pkgName, "use vendor", c.Vendored) if c.Vendored { - utils.Logger.Info("Using dependency manager to import package", "package", pkgName) - - if depPath == "" { - utils.Logger.Error("Build: Vendor folder found, but the `dep` tool was not found, " + - "if you use a different vendoring (package management) tool please add the following packages by hand, " + - "or install the `dep` tool into your gopath by doing a `go get -u github.com/golang/dep/cmd/dep`. " + - "For more information and usage of the tool please see http://github.com/golang/dep") - utils.Logger.Error("Missing package", "package", pkgName) - return fmt.Errorf("Missing package %s", pkgName) - } - // Check to see if the package exists locally - _, err := build.Import(pkgName, c.AppPath, build.FindOnly) - if err != nil { - getCmd = exec.Command(depPath, "ensure", "-add", pkgName) - } else { - getCmd = exec.Command(depPath, "ensure", "-update", pkgName) - } - - } else { - utils.Logger.Info("No vendor folder detected, not using dependency manager to import package", "package", pkgName) - getCmd = exec.Command(c.GoCmd, "get", "-u", pkgName) + goModCmd := exec.Command("go", "mod", "tidy") + utils.CmdInit(goModCmd, c.AppPath) + return nil } - - utils.CmdInit(getCmd, c.AppPath) - utils.Logger.Info("Go get command ", "exec", getCmd.Path, "dir", getCmd.Dir, "args", getCmd.Args, "env", getCmd.Env, "package", pkgName) - output, err := getCmd.CombinedOutput() - if err != nil { - fmt.Printf("Error stack %v\n", logger.NewCallStack()) - utils.Logger.Error("Failed to import package", "error", err, "gopath", build.Default.GOPATH, "GO-ROOT", build.Default.GOROOT, "output", string(output)) - } - return err + //utils.Logger.Info("Using dependency manager to import package", "package", pkgName) + // + //// Check to see if the package exists locally + //_, err := build.Import(pkgName, c.AppPath, build.FindOnly) + //if err != nil { + // getCmd = exec.Command(depPath, "ensure", "-add", pkgName) + //} else { + // getCmd = exec.Command(depPath, "ensure", "-update", pkgName) + //} + // + // + //} else { + // utils.Logger.Info("No vendor folder detected, not using dependency manager to import package", "package", pkgName) + // getCmd = exec.Command(c.GoCmd, "get", "-u", pkgName) + //} + // + //utils.CmdInit(getCmd, c.AppPath) + //utils.Logger.Info("Go get command ", "exec", getCmd.Path, "dir", getCmd.Dir, "args", getCmd.Args, "env", getCmd.Env, "package", pkgName) + //output, err := getCmd.CombinedOutput() + //if err != nil { + // fmt.Printf("Error stack %v\n", logger.NewCallStack()) + // utils.Logger.Error("Failed to import package", "error", err, "gopath", build.Default.GOPATH, "GO-ROOT", build.Default.GOROOT, "output", string(output)) + //} + return nil } } // lookup and set Go related variables -func (c *CommandConfig) InitGoPaths() { +func (c *CommandConfig) InitGoPathsOld() { utils.Logger.Info("InitGoPaths") // lookup go path c.GoPath = build.Default.GOPATH @@ -276,17 +377,21 @@ func (c *CommandConfig) InitGoPaths() { } } - utils.Logger.Info("Source root", "path", c.SrcRoot, "cwd", workingDir, "gopath", c.GoPath, "bestpath", bestpath) + utils.Logger.Info("Source root", "path", c.SrcRoot, "cwd", workingDir, "gopath", c.GoPath, "bestpath",bestpath) if len(c.SrcRoot) == 0 && len(bestpath) > 0 { c.SrcRoot = bestpath } // If source root is empty and this isn't a version then skip it if len(c.SrcRoot) == 0 { - if c.Index != VERSION { - utils.Logger.Fatal("Abort: could not create a Revel application outside of GOPATH.") + if c.Index == NEW { + c.SrcRoot = c.New.ImportPath + } else { + if c.Index != VERSION { + utils.Logger.Fatal("Abort: could not create a Revel application outside of GOPATH.") + } + return } - return } // set go src path @@ -299,14 +404,14 @@ func (c *CommandConfig) InitGoPaths() { // Sets the versions on the command config func (c *CommandConfig) SetVersions() (err error) { c.CommandVersion, _ = ParseVersion(cmd.Version) - _, revelPath, err := utils.FindSrcPaths(c.ImportPath, RevelImportPath, c.PackageResolver) + pathMap, err := utils.FindSrcPaths(c.AppPath, []string{RevelImportPath}, c.PackageResolver) if err == nil { - utils.Logger.Info("Fullpath to revel", "dir", revelPath) + utils.Logger.Info("Fullpath to revel", "dir", pathMap[RevelImportPath]) fset := token.NewFileSet() // positions are relative to fset - versionData, err := ioutil.ReadFile(filepath.Join(revelPath, RevelImportPath, "version.go")) + versionData, err := ioutil.ReadFile(filepath.Join(pathMap[RevelImportPath], "version.go")) if err != nil { - utils.Logger.Error("Failed to find Revel version:", "error", err, "path", revelPath) + utils.Logger.Error("Failed to find Revel version:", "error", err, "path", pathMap[RevelImportPath]) } // Parse src but stop after processing the imports. diff --git a/model/revel_container.go b/model/revel_container.go index f15a665..0647208 100644 --- a/model/revel_container.go +++ b/model/revel_container.go @@ -4,13 +4,12 @@ package model import ( "github.com/revel/cmd/utils" "github.com/revel/config" - "go/build" - "errors" "fmt" "path/filepath" "sort" "strings" + "golang.org/x/tools/go/packages" ) type ( @@ -65,7 +64,11 @@ type ( CookieSecure bool // True if cookie is secure SecretStr string // The secret string MimeConfig *config.Context // The mime configuration - ModulePathMap map[string]string // The module path map + ModulePathMap map[string]*ModuleInfo // The module path map + } + ModuleInfo struct { + ImportPath string + Path string } WrappedRevelCallback struct { @@ -96,30 +99,22 @@ var RevelModulesImportPath = "github.com/revel/modules" // This function returns a container object describing the revel application // eventually this type of function will replace the global variables. -func NewRevelPaths(mode, importPath, srcPath string, callback RevelCallback) (rp *RevelContainer, err error) { - rp = &RevelContainer{ModulePathMap: map[string]string{}} +func NewRevelPaths(mode, importPath, appSrcPath string, callback RevelCallback) (rp *RevelContainer, err error) { + rp = &RevelContainer{ModulePathMap: map[string]*ModuleInfo{}} // Ignore trailing slashes. rp.ImportPath = strings.TrimRight(importPath, "/") - rp.SourcePath = srcPath + rp.SourcePath = appSrcPath rp.RunMode = mode - // If the SourcePath is not specified, find it using build.Import. - var revelSourcePath string // may be different from the app source path - if rp.SourcePath == "" { - rp.SourcePath, revelSourcePath, err = utils.FindSrcPaths(importPath, RevelImportPath, callback.PackageResolver) - if err != nil { - return - } - } else { - // If the SourcePath was specified, assume both Revel and the app are within it. - rp.SourcePath = filepath.Clean(rp.SourcePath) - revelSourcePath = rp.SourcePath + // We always need to determine the paths for files + pathMap, err := utils.FindSrcPaths(appSrcPath, []string{importPath+"/app", RevelImportPath}, callback.PackageResolver) + if err != nil { + return } - + rp.AppPath, rp.RevelPath = pathMap[importPath], pathMap[RevelImportPath] // Setup paths for application - rp.RevelPath = filepath.Join(revelSourcePath, filepath.FromSlash(RevelImportPath)) - rp.BasePath = filepath.Join(rp.SourcePath, filepath.FromSlash(importPath)) - rp.PackageInfo.Vendor = utils.Exists(filepath.Join(rp.BasePath, "vendor")) + rp.BasePath = rp.SourcePath + rp.PackageInfo.Vendor = utils.Exists(filepath.Join(rp.BasePath, "go.mod")) rp.AppPath = filepath.Join(rp.BasePath, "app") // Sanity check , ensure app and conf paths exist @@ -188,6 +183,7 @@ func NewRevelPaths(mode, importPath, srcPath string, callback RevelCallback) (rp rp.SecretStr = rp.Config.StringDefault("app.secret", "") callback.FireEvent(REVEL_BEFORE_MODULES_LOADED, nil) + utils.Logger.Info("Loading modules") if err := rp.loadModules(callback); err != nil { return rp, err } @@ -248,9 +244,10 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) { // Adds a module paths to the container object func (rp *RevelContainer) addModulePaths(name, importPath, modulePath string) { + utils.Logger.Info("Adding module path","name", name,"import path", importPath,"system path", modulePath) if codePath := filepath.Join(modulePath, "app"); utils.DirExists(codePath) { rp.CodePaths = append(rp.CodePaths, codePath) - rp.ModulePathMap[name] = modulePath + rp.ModulePathMap[name] = &ModuleInfo{importPath, modulePath} if viewsPath := filepath.Join(modulePath, "app", "views"); utils.DirExists(viewsPath) { rp.TemplatePaths = append(rp.TemplatePaths, viewsPath) } @@ -273,13 +270,21 @@ func (rp *RevelContainer) ResolveImportPath(importPath string) (string, error) { if rp.Packaged { return filepath.Join(rp.SourcePath, importPath), nil } + config := &packages.Config{ + Mode: packages.LoadSyntax, + Dir:rp.AppPath, + } - modPkg, err := build.Import(importPath, rp.AppPath, build.FindOnly) + pkgs, err := packages.Load(config, importPath) + if len(pkgs)==0 { + return "", errors.New("No packages found for import " + importPath +" using app path "+ rp.AppPath) + } +// modPkg, err := build.Import(importPath, rp.AppPath, build.FindOnly) if err != nil { return "", err } - if rp.PackageInfo.Vendor && !strings.HasPrefix(modPkg.Dir,rp.BasePath) { - return "", fmt.Errorf("Module %s was found outside of path %s.",importPath, modPkg.Dir) + if len(pkgs[0].GoFiles)>0 { + return filepath.Dir(pkgs[0].GoFiles[0]), nil } - return modPkg.Dir, nil + return pkgs[0].PkgPath, errors.New("No files found in import path " + importPath) } diff --git a/parser/imports.go b/parser/imports.go index 150c6ed..8f40285 100644 --- a/parser/imports.go +++ b/parser/imports.go @@ -42,13 +42,14 @@ func addImports(imports map[string]string, decl ast.Decl, srcDir string) { // 2. Exempt the standard library; their directories always match the package name. // 3. Can use build.FindOnly and then use parser.ParseDir with mode PackageClauseOnly if pkgAlias == "" { + utils.Logger.Debug("Reading from build", "path", fullPath, "srcPath", srcDir, "gopath", build.Default.GOPATH) pkg, err := build.Import(fullPath, srcDir, 0) if err != nil { // We expect this to happen for apps using reverse routing (since we // have not yet generated the routes). Don't log that. if !strings.HasSuffix(fullPath, "/app/routes") { - utils.Logger.Error("Could not find import:", "path", fullPath, "srcPath", srcDir, "error", err) + utils.Logger.Warn("Could not find import:", "path", fullPath, "srcPath", srcDir, "error", err) } continue } else { diff --git a/parser/utils.go b/parser/utils.go new file mode 100644 index 0000000..8239d15 --- /dev/null +++ b/parser/utils.go @@ -0,0 +1,18 @@ +package parser + +import ( + //"golang.org/x/tools/go/packages" + //"github.com/revel/cmd/utils" +) +//import "golang.org/x/tools/go/packages" +// +//func GetPackage(appPath, importPath string) { +// config := &packages.Config{ +// Mode: packages.NeedName | packages.NeedFiles, +// Dir:appPath, +// } +// +// pkgs, err := packages.Load(config, []string{importPath}) +// utils.Logger.Info("Loaded packegs ", "len results", len(pkgs), "error",err) +// +//} diff --git a/parser2/read.go b/parser2/read.go new file mode 100644 index 0000000..ed30a69 --- /dev/null +++ b/parser2/read.go @@ -0,0 +1,123 @@ +package parser2 + +import ( + "go/ast" + "go/token" + + "github.com/revel/cmd/model" + "golang.org/x/tools/go/packages" + "github.com/revel/cmd/utils" + "github.com/pkg/errors" + "golang.org/x/tools/go/ssa/interp/testdata/src/strings" +) +func ProcessSource(revelContainer *model.RevelContainer) (sourceInfo *model.SourceInfo, compileError error) { + utils.Logger.Info("ProcessSource") + // Combine packages for modules and app and revel + allPackages := []string{revelContainer.ImportPath+"/app/controllers/...",model.RevelImportPath} + for _,module := range revelContainer.ModulePathMap { + allPackages = append(allPackages,module.ImportPath+"/app/controllers/...") + } + + config := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.LoadTypes | packages.NeedTypes | packages.NeedSyntax , //| packages.NeedImports | + // packages.NeedTypes, // packages.LoadTypes | packages.NeedSyntax | packages.NeedTypesInfo, + //packages.LoadSyntax | packages.NeedDeps, + Dir:revelContainer.AppPath, + } + utils.Logger.Info("Before ","apppath", config.Dir,"paths",allPackages) + pkgs, err := packages.Load(config, allPackages...) + utils.Logger.Info("***Loaded packegs ", "len results", len(pkgs), "error",err) + // Lets see if we can output all the path names + //packages.Visit(pkgs,func(p *packages.Package) bool{ + // println("Got pre",p.ID) + // return true + //}, func(p *packages.Package) { + //}) + counter := 0 + for _, p := range pkgs { + utils.Logger.Info("Errores","error",p.Errors, "id",p.ID) + //for _,g := range p.GoFiles { + // println("File", g) + //} + //for _, t:= range p.Syntax { + // utils.Logger.Info("File","name",t.Name) + //} + println("package typoe fouhnd ",p.Types.Name()) + imports := map[string]string{} + + for _,s := range p.Syntax { + println("File ",s.Name.Name ) + for _, decl := range file.Decls { + if decl.Tok == token.IMPORT { + } + } + } + //p.Fset.Iterate(func(file *token.File) bool{ + // + // // utils.Logger.Info("Output","Found file", p.ID," AND NAME ", f.Name()) + // // For each declaration in the source file... + // //for _, decl := range file.Decls { + // // addImports(imports, decl, pkgPath) + // //} + // counter ++ + // return true + //}) + } + + err = errors.New("Incompleted") + println("*******************", counter) + utils.Logger.Panic("Not implemented") + return +} + + + +// Add imports to the map from the source dir +func addImports(imports map[string]string, decl ast.Decl, srcDir string) { + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + return + } + + if genDecl.Tok != token.IMPORT { + return + } + + for _, spec := range genDecl.Specs { + importSpec := spec.(*ast.ImportSpec) + var pkgAlias string + if importSpec.Name != nil { + pkgAlias = importSpec.Name.Name + if pkgAlias == "_" { + continue + } + } + quotedPath := importSpec.Path.Value // e.g. "\"sample/app/models\"" + fullPath := quotedPath[1 : len(quotedPath)-1] // Remove the quotes + + // If the package was not aliased (common case), we have to import it + // to see what the package name is. + // TODO: Can improve performance here a lot: + // 1. Do not import everything over and over again. Keep a cache. + // 2. Exempt the standard library; their directories always match the package name. + // 3. Can use build.FindOnly and then use parser.ParseDir with mode PackageClauseOnly + if pkgAlias == "" { + + utils.Logger.Debug("Reading from build", "path", fullPath, "srcPath", srcDir, "gopath", build.Default.GOPATH) + pkg, err := build.Import(fullPath, srcDir, 0) + if err != nil { + // We expect this to happen for apps using reverse routing (since we + // have not yet generated the routes). Don't log that. + if !strings.HasSuffix(fullPath, "/app/routes") { + utils.Logger.Warn("Could not find import:", "path", fullPath, "srcPath", srcDir, "error", err) + } + continue + } else { + utils.Logger.Debug("Found package in dir", "dir", pkg.Dir, "name", pkg.ImportPath) + } + pkgAlias = pkg.Name + } + + imports[pkgAlias] = fullPath + } +} \ No newline at end of file diff --git a/revel/build.go b/revel/build.go index 352fc0f..e929114 100644 --- a/revel/build.go +++ b/revel/build.go @@ -13,7 +13,6 @@ import ( "github.com/revel/cmd/harness" "github.com/revel/cmd/model" "github.com/revel/cmd/utils" - "go/build" ) var cmdBuild = &Command{ @@ -153,10 +152,10 @@ func buildCopyModules(c *model.CommandConfig, revel_paths *model.RevelContainer, destPath := filepath.Join(c.Build.TargetPath, "src") // Find all the modules used and copy them over. config := revel_paths.Config.Raw() - modulePaths := make(map[string]string) // import path => filesystem path // We should only copy over the section of options what the build is targeted for // We will default to prod + moduleImportList := []string{} for _, section := range config.Sections() { // If the runmode is defined we will only import modules defined for that run mode if c.Build.Mode != "" && c.Build.Mode != section { @@ -171,14 +170,15 @@ func buildCopyModules(c *model.CommandConfig, revel_paths *model.RevelContainer, if moduleImportPath == "" { continue } + moduleImportList =append(moduleImportList,moduleImportPath) - modPkg, err := build.Import(moduleImportPath, revel_paths.RevelPath, build.FindOnly) - if err != nil { - utils.Logger.Fatalf("Failed to load module %s (%s): %s", key[len("module."):], c.ImportPath, err) - } - modulePaths[moduleImportPath] = modPkg.Dir } } + modulePaths, err := utils.FindSrcPaths(c.AppPath, moduleImportList, c.PackageResolver) + + if err != nil { + utils.Logger.Fatalf("Failed to load modules ", "error", err) + } // Copy the the paths for each of the modules for importPath, fsPath := range modulePaths { diff --git a/revel/new.go b/revel/new.go index 84f848e..051f0c0 100644 --- a/revel/new.go +++ b/revel/new.go @@ -76,56 +76,15 @@ func newApp(c *model.CommandConfig) (err error) { return utils.NewBuildError("Abort: Unable to create app path.", "path", c.AppPath) } - if c.New.Vendored { - utils.Logger.Info("Creating a new vendor app") - - vendorPath := filepath.Join(c.AppPath, "vendor") - if !utils.DirExists(vendorPath) { - - if err := os.MkdirAll(vendorPath, os.ModePerm); err != nil { - return utils.NewBuildError("Failed to create "+vendorPath, "error", err) - } - } - - // In order for dep to run there needs to be a source file in the folder - tempPath := filepath.Join(c.AppPath, "tmp") - utils.Logger.Info("Checking for temp folder for source code", "path", tempPath) - if !utils.DirExists(tempPath) { - if err := os.MkdirAll(tempPath, os.ModePerm); err != nil { - return utils.NewBuildIfError(err, "Failed to create "+vendorPath) - } - - if err = utils.GenerateTemplate(filepath.Join(tempPath, "main.go"), NEW_MAIN_FILE, nil); err != nil { - return utils.NewBuildIfError(err, "Failed to create main file "+vendorPath) - } - } - - // Create a package template file if it does not exist - packageFile := filepath.Join(c.AppPath, "Gopkg.toml") - utils.Logger.Info("Checking for Gopkg.toml", "path", packageFile) - if !utils.Exists(packageFile) { - utils.Logger.Info("Generating Gopkg.toml", "path", packageFile) - if err := utils.GenerateTemplate(packageFile, VENDOR_GOPKG, nil); err != nil { - return utils.NewBuildIfError(err, "Failed to generate template") - } - } else { - utils.Logger.Info("Package file exists in skeleto, skipping adding") - } - - getCmd := exec.Command("dep", "ensure", "-v") - utils.CmdInit(getCmd, c.AppPath) - - utils.Logger.Info("Exec:", "args", getCmd.Args, "env", getCmd.Env, "workingdir",getCmd.Dir) - getOutput, err := getCmd.CombinedOutput() - if err != nil { - return utils.NewBuildIfError(err, string(getOutput)) - } + if len(c.New.Package)>0 { + c.New.Vendored = true } // checking and setting application if err = setApplicationPath(c); err != nil { return err } + // At this point the versions can be set c.SetVersions() @@ -136,12 +95,8 @@ func newApp(c *model.CommandConfig) (err error) { // Rerun the dep tool if vendored if c.New.Vendored { - getCmd := exec.Command("dep", "ensure", "-v") - utils.CmdInit(getCmd, c.AppPath) - utils.Logger.Info("Exec:", "args", getCmd.Args) - getOutput, err := getCmd.CombinedOutput() - if err != nil { - utils.Logger.Fatal(string(getOutput)) + if err=createModVendor(c); err!=nil { + return } } @@ -156,6 +111,70 @@ func newApp(c *model.CommandConfig) (err error) { return } +func createModVendor(c *model.CommandConfig) (err error) { + + utils.Logger.Info("Creating a new mod app") + goModCmd := exec.Command("go", "mod", "init", filepath.Join(c.New.Package,c.AppName)) + + + utils.CmdInit(goModCmd, c.AppPath) + + utils.Logger.Info("Exec:", "args", goModCmd.Args, "env", goModCmd.Env, "workingdir",goModCmd.Dir) + getOutput, err := goModCmd.CombinedOutput() + if err != nil { + return utils.NewBuildIfError(err, string(getOutput)) + } + return +} + +func createDepVendor(c *model.CommandConfig) (err error) { + + utils.Logger.Info("Creating a new vendor app") + + vendorPath := filepath.Join(c.AppPath, "vendor") + if !utils.DirExists(vendorPath) { + + if err := os.MkdirAll(vendorPath, os.ModePerm); err != nil { + return utils.NewBuildError("Failed to create "+vendorPath, "error", err) + } + } + + // In order for dep to run there needs to be a source file in the folder + tempPath := filepath.Join(c.AppPath, "tmp") + utils.Logger.Info("Checking for temp folder for source code", "path", tempPath) + if !utils.DirExists(tempPath) { + if err := os.MkdirAll(tempPath, os.ModePerm); err != nil { + return utils.NewBuildIfError(err, "Failed to create "+vendorPath) + } + + if err = utils.GenerateTemplate(filepath.Join(tempPath, "main.go"), NEW_MAIN_FILE, nil); err != nil { + return utils.NewBuildIfError(err, "Failed to create main file "+vendorPath) + } + } + + // Create a package template file if it does not exist + packageFile := filepath.Join(c.AppPath, "Gopkg.toml") + utils.Logger.Info("Checking for Gopkg.toml", "path", packageFile) + if !utils.Exists(packageFile) { + utils.Logger.Info("Generating Gopkg.toml", "path", packageFile) + if err := utils.GenerateTemplate(packageFile, VENDOR_GOPKG, nil); err != nil { + return utils.NewBuildIfError(err, "Failed to generate template") + } + } else { + utils.Logger.Info("Package file exists in skeleto, skipping adding") + } + + getCmd := exec.Command("dep", "ensure", "-v") + utils.CmdInit(getCmd, c.AppPath) + + utils.Logger.Info("Exec:", "args", getCmd.Args, "env", getCmd.Env, "workingdir",getCmd.Dir) + getOutput, err := getCmd.CombinedOutput() + if err != nil { + return utils.NewBuildIfError(err, string(getOutput)) + } + return +} + // Used to generate a new secret key const alphaNumeric = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" diff --git a/revel/new_test.go b/revel/new_test.go index c1d3bca..31aa2b4 100644 --- a/revel/new_test.go +++ b/revel/new_test.go @@ -55,12 +55,12 @@ func TestNewVendor(t *testing.T) { a := assert.New(t) gopath := setup("revel-test-new-vendor", a) precall := func(c *model.CommandConfig) { - c.New.Vendored = true + c.New.DepVendored = true } t.Run("New", func(t *testing.T) { a := assert.New(t) c := newApp("onlyone/v/a", model.NEW, precall, a) - c.New.Vendored = true + c.New.DepVendored = true a.Nil(main.Commands[model.NEW].RunWith(c), "New failed") }) t.Run("Test", func(t *testing.T) { diff --git a/revel/revel.go b/revel/revel.go index dd95b43..1538066 100644 --- a/revel/revel.go +++ b/revel/revel.go @@ -101,7 +101,7 @@ func main() { println("Revel executing:", command.Short) // Setting go paths - c.InitGoPaths() + // c.InitGoPaths() // Setup package resolver c.InitPackageResolver() diff --git a/revel/run.go b/revel/run.go index c7bd5ee..a149afd 100644 --- a/revel/run.go +++ b/revel/run.go @@ -126,7 +126,7 @@ func runApp(c *model.CommandConfig) (err error) { c.Run.Mode = "dev" } - revel_path, err := model.NewRevelPaths(c.Run.Mode, c.ImportPath, "", model.NewWrappedRevelCallback(nil, c.PackageResolver)) + revel_path, err := model.NewRevelPaths(c.Run.Mode, c.ImportPath, c.AppPath, model.NewWrappedRevelCallback(nil, c.PackageResolver)) if err != nil { return utils.NewBuildIfError(err, "Revel paths") } diff --git a/revel/test.go b/revel/test.go index d1ec415..8465f6b 100644 --- a/revel/test.go +++ b/revel/test.go @@ -258,7 +258,7 @@ func getTestsList(baseURL string) (*[]tests.TestSuiteDesc, error) { func runTestSuites(paths *model.RevelContainer, baseURL, resultPath string, testSuites *[]tests.TestSuiteDesc) (*[]tests.TestSuiteResult, bool) { // We can determine the testsuite location by finding the test module and extracting the data from it - resultFilePath := filepath.Join(paths.ModulePathMap["testrunner"], "app", "views", "TestRunner/SuiteResult.html") + resultFilePath := filepath.Join(paths.ModulePathMap["testrunner"].Path, "app", "views", "TestRunner/SuiteResult.html") var ( overallSuccess = true diff --git a/revel/version.go b/revel/version.go index 1000693..af0a43e 100644 --- a/revel/version.go +++ b/revel/version.go @@ -77,7 +77,7 @@ func (v *VersionCommand) RunWith(c *model.CommandConfig) (err error) { versionInfo, needsUpdates = v.doRepoCheck(x==0) } - fmt.Println(versionInfo) + fmt.Printf("%s\n\nGo Location:%s\n\n",versionInfo,c.GoCmd) cmd := exec.Command(c.GoCmd, "version") cmd.Stdout = os.Stdout if e := cmd.Start(); e != nil { @@ -234,28 +234,20 @@ func (v *VersionCommand) updateLocalVersions() { v.cmdVersion.BuildDate = cmd.BuildDate v.cmdVersion.MinGoVersion = cmd.MinimumGoVersion - var modulePath, revelPath string - _, revelPath, err := utils.FindSrcPaths(v.Command.ImportPath, model.RevelImportPath, v.Command.PackageResolver) + pathMap, err := utils.FindSrcPaths(v.Command.AppPath,[]string{model.RevelImportPath,model.RevelModulesImportPath}, v.Command.PackageResolver) if err != nil { - utils.Logger.Warn("Unable to extract version information from Revel library", "error,err") + utils.Logger.Warn("Unable to extract version information from Revel library", "path",pathMap[model.RevelImportPath], "error",err) return } - revelPath = revelPath + model.RevelImportPath - utils.Logger.Info("Fullpath to revel", "dir", revelPath) - v.revelVersion, err = v.versionFromFilepath(revelPath) + utils.Logger.Info("Fullpath to revel", "dir", pathMap[model.RevelModulesImportPath]) + v.revelVersion, err = v.versionFromFilepath(pathMap[model.RevelModulesImportPath]) if err != nil { utils.Logger.Warn("Unable to extract version information from Revel", "error,err") } - _, modulePath, err = utils.FindSrcPaths(v.Command.ImportPath, model.RevelModulesImportPath, v.Command.PackageResolver) + v.modulesVersion, err = v.versionFromFilepath(pathMap[model.RevelModulesImportPath]) if err != nil { - utils.Logger.Warn("Unable to extract version information from Revel library", "error,err") - return - } - modulePath = modulePath + model.RevelModulesImportPath - v.modulesVersion, err = v.versionFromFilepath(modulePath) - if err != nil { - utils.Logger.Warn("Unable to extract version information from Revel Modules", "error", err) + utils.Logger.Warn("Unable to extract version information from Revel Modules", "path", pathMap[model.RevelModulesImportPath], "error", err) } return diff --git a/utils/file.go b/utils/file.go index cb714ee..c070c35 100644 --- a/utils/file.go +++ b/utils/file.go @@ -6,13 +6,13 @@ import ( "compress/gzip" "errors" "fmt" - "go/build" "html/template" "io" "io/ioutil" "os" "path/filepath" "strings" + "golang.org/x/tools/go/packages" ) // DirExists returns true if the given path exists and is a directory. @@ -320,49 +320,63 @@ func Empty(dirname string) bool { } // Find the full source dir for the import path, uses the build.Default.GOPATH to search for the directory -func FindSrcPaths(appImportPath, revelImportPath string, packageResolver func(pkgName string) error) (appSourcePath, revelSourcePath string, err error) { - var ( - gopaths = filepath.SplitList(build.Default.GOPATH) - goroot = build.Default.GOROOT - ) - - if len(gopaths) == 0 { - err = errors.New("GOPATH environment variable is not set. " + - "Please refer to http://golang.org/doc/code.html to configure your Go environment.") - return - } - - if ContainsString(gopaths, goroot) { - err = fmt.Errorf("GOPATH (%s) must not include your GOROOT (%s). "+ - "Please refer to http://golang.org/doc/code.html to configure your Go environment. ", - build.Default.GOPATH, goroot) - return - - } - - appPkgDir := "" - appPkgSrcDir := "" - if len(appImportPath)>0 { - Logger.Info("Seeking app package","app",appImportPath) - appPkg, err := build.Import(appImportPath, "", build.FindOnly) - if err != nil { - err = fmt.Errorf("Failed to import " + appImportPath + " with error %s", err.Error()) - return "","",err +func FindSrcPaths(appPath string, packageList []string, packageResolver func(pkgName string) error) (sourcePathsmap map[string]string, err error) { + sourcePathsmap, missingList,err := findSrcPaths(appPath,packageList) + if err != nil && packageResolver != nil { + for _, item := range missingList { + if err = packageResolver(item); err != nil { + return + } } - appPkgDir,appPkgSrcDir =appPkg.Dir, appPkg.SrcRoot + sourcePathsmap,missingList,err = findSrcPaths(appPath,packageList) } - Logger.Info("Seeking remote package","using",appImportPath, "remote",revelImportPath) - revelPkg, err := build.Default.Import(revelImportPath, appPkgDir, build.FindOnly) - if err != nil { - Logger.Info("Resolved called Seeking remote package","using",appImportPath, "remote",revelImportPath) - packageResolver(revelImportPath) - revelPkg, err = build.Import(revelImportPath, appPkgDir, build.FindOnly) - if err != nil { - err = fmt.Errorf("Failed to find Revel with error: %s", err.Error()) - return + if err!=nil && len(missingList)>0 { + for _, missing := range missingList { + Logger.Error("Unable to import this package", "package", missing) } } - revelSourcePath, appSourcePath = revelPkg.Dir[:len(revelPkg.Dir)-len(revelImportPath)], appPkgSrcDir + return +} +var NO_APP_FOUND = errors.New("No app found") +var NO_REVEL_FOUND = errors.New("No revel found") + +// Find the full source dir for the import path, uses the build.Default.GOPATH to search for the directory +func findSrcPaths(appPath string, packagesList []string) (sourcePathsmap map[string]string, missingList[] string, err error) { + // Use packages to fetch + // by not specifying env, we will use the default env + config := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles, + Dir:appPath, + } + sourcePathsmap = map[string]string{} + + pkgs, err := packages.Load(config, packagesList...) + Logger.Info("Loaded packegs ", "len results", len(pkgs), "error",err) + for _, packageName := range packagesList { + found := false + log:= Logger.New("seeking",packageName) + for _, pck := range pkgs { + log.Info("Found package","package",pck.ID) + if pck.ID == packageName { + if pck.Errors!=nil && len(pck.Errors)>0 { + Logger.Info("Error ", "count", len(pck.Errors), "App Import Path", pck.ID,"errors",pck.Errors) + + } + //a,_ := pck.MarshalJSON() + Logger.Info("Found ", "count", len(pck.GoFiles), "App Import Path", pck.ID) + sourcePathsmap[packageName] = filepath.Dir(pck.GoFiles[0]) + found = true + } + } + if !found { + if packageName == "github.com/revel/revel" { + err = NO_REVEL_FOUND + } else { + err = NO_APP_FOUND + } + missingList = append(missingList,packageName) + } + } return }