From f2b54f5a6932eb96c1f2cc9891f89025ecd75cf9 Mon Sep 17 00:00:00 2001 From: "notzippy@gmail.com" Date: Sun, 26 Apr 2020 08:58:58 -0700 Subject: [PATCH] Updated sourceinfo Added packagepathmap to the SourceInfo, this in turn allows the RevelCLI app command to pass the source paths directly to Revel directly Added default to build to be "target" of the current folder Renamed source processor --- go.mod | 2 + harness/app.go | 5 +- harness/build.go | 13 +-- harness/harness.go | 14 ++- model/source_info.go | 2 + parser2/source_info_processor.go | 3 + parser2/{read.go => source_processor.go} | 119 +++++------------------ revel/build.go | 20 ++-- revel/clean.go | 3 + revel/package.go | 5 +- revel/run.go | 8 +- revel/test.go | 13 ++- utils/file.go | 8 +- 13 files changed, 90 insertions(+), 125 deletions(-) rename parser2/{read.go => source_processor.go} (61%) diff --git a/go.mod b/go.mod index a6e1c32..e6ee7ee 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/revel/config v0.21.0 github.com/revel/log15 v2.11.20+incompatible + github.com/revel/modules v0.21.0 // indirect github.com/revel/pathtree v0.0.0-20140121041023-41257a1839e9 // indirect github.com/revel/revel v0.21.0 github.com/stretchr/testify v1.4.0 @@ -21,6 +22,7 @@ require ( golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect golang.org/x/tools v0.0.0-20200219054238-753a1d49df85 gopkg.in/fsnotify/fsnotify.v1 v1.4.7 + gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/stack.v0 v0.0.0-20141108040640-9b43fcefddd0 gopkg.in/stretchr/testify.v1 v1.2.2 // indirect diff --git a/harness/app.go b/harness/app.go index 3b855b8..3b5475a 100644 --- a/harness/app.go +++ b/harness/app.go @@ -24,12 +24,13 @@ type App struct { BinaryPath string // Path to the app executable Port int // Port to pass as a command line argument. cmd AppCmd // The last cmd returned. + PackagePathMap map[string]string // Package to directory path map Paths *model.RevelContainer } // NewApp returns app instance with binary path in it -func NewApp(binPath string, paths *model.RevelContainer) *App { - return &App{BinaryPath: binPath, Paths: paths, Port: paths.HTTPPort} +func NewApp(binPath string, paths *model.RevelContainer, packagePathMap map[string]string) *App { + return &App{BinaryPath: binPath, Paths: paths, Port: paths.HTTPPort, PackagePathMap:packagePathMap} } // Cmd returns a command to run the app server using the current configuration. diff --git a/harness/build.go b/harness/build.go index 51adb64..729fd15 100644 --- a/harness/build.go +++ b/harness/build.go @@ -116,13 +116,8 @@ func Build(c *model.CommandConfig, paths *model.RevelContainer) (_ *App, err err } } - pkg, err := build.Default.Import(paths.ImportPath, "", build.FindOnly) - if err != nil { - return - } - - // Binary path is a combination of $GOBIN/revel.d directory, app's import path and its name. - binName := filepath.Join(pkg.BinDir, "revel.d", paths.ImportPath, filepath.Base(paths.BasePath)) + // Binary path is a combination of BasePath/target directory, app's import path and its name. + binName := filepath.Join(paths.BasePath, "target", paths.ImportPath, filepath.Base(paths.BasePath)) // Change binary path for Windows build goos := runtime.GOOS @@ -196,13 +191,13 @@ func Build(c *model.CommandConfig, paths *model.RevelContainer) (_ *App, err err "GOPATH="+gopath, ) utils.CmdInit(buildCmd, c.AppPath) - utils.Logger.Info("Exec:", "args", buildCmd.Args) + utils.Logger.Info("Exec:", "args", buildCmd.Args,"working dir", buildCmd.Dir) output, err := buildCmd.CombinedOutput() // If the build succeeded, we're done. if err == nil { utils.Logger.Info("Build successful continuing") - return NewApp(binName, paths), nil + return NewApp(binName, paths,sourceInfo.PackageMap), nil } // Since there was an error, capture the output in case we need to report it diff --git a/harness/harness.go b/harness/harness.go index bdf8002..fd6397c 100644 --- a/harness/harness.go +++ b/harness/harness.go @@ -34,6 +34,7 @@ import ( "html/template" "io/ioutil" "sync" + "encoding/json" ) var ( @@ -161,6 +162,7 @@ func NewHarness(c *model.CommandConfig, paths *model.RevelContainer, runMode str addr := paths.HTTPAddr port := paths.Config.IntDefault("harness.port", 0) scheme := "http" + if paths.HTTPSsl { scheme = "https" } @@ -229,7 +231,17 @@ func (h *Harness) Refresh() (err *utils.SourceError) { if h.useProxy { h.app.Port = h.port - if err2 := h.app.Cmd(h.runMode).Start(h.config); err2 != nil { + runMode := h.runMode + if !h.config.HistoricMode { + // Recalulate run mode based on the config + var paths []byte + if len(h.app.PackagePathMap)>0 { + paths, _ = json.Marshal(h.app.PackagePathMap) + } + runMode = fmt.Sprintf(`{"mode":"%s", "specialUseFlag":%v,"packagePathMap":%s}`, h.app.Paths.RunMode, h.config.Verbose, string(paths)) + + } + if err2 := h.app.Cmd(runMode).Start(h.config); err2 != nil { utils.Logger.Error("Could not start application", "error", err2) if err,k :=err2.(*utils.SourceError);k { return err diff --git a/model/source_info.go b/model/source_info.go index c7ca05f..88ed26f 100644 --- a/model/source_info.go +++ b/model/source_info.go @@ -29,6 +29,8 @@ type SourceInfo struct { controllerSpecs []*TypeInfo // testSuites list the types that constitute the set of application tests. testSuites []*TypeInfo + // packageMap a map of import to system directory (if available) + PackageMap map[string]string } // TypesThatEmbed returns all types that (directly or indirectly) embed the diff --git a/parser2/source_info_processor.go b/parser2/source_info_processor.go index 1aca9ca..6d8a1c0 100644 --- a/parser2/source_info_processor.go +++ b/parser2/source_info_processor.go @@ -7,6 +7,7 @@ import ( "go/ast" "go/token" "strings" + "path/filepath" ) type ( @@ -32,6 +33,8 @@ func (s *SourceInfoProcessor) processPackage(p *packages.Package) (sourceInfo *m ) for _,tree := range p.Syntax { for _, decl := range tree.Decls { + s.sourceProcessor.packageMap[p.PkgPath] = filepath.Dir(p.Fset.Position(decl.Pos()).Filename) + //println("*** checking", p.Fset.Position(decl.Pos()).Filename) spec, found := s.getStructTypeDecl(decl, p.Fset) if found { if isController || isTest { diff --git a/parser2/read.go b/parser2/source_processor.go similarity index 61% rename from parser2/read.go rename to parser2/source_processor.go index 48b35a1..2811da1 100644 --- a/parser2/read.go +++ b/parser2/source_processor.go @@ -3,13 +3,11 @@ package parser2 import ( "go/ast" "go/token" - "github.com/revel/cmd/model" "golang.org/x/tools/go/packages" "github.com/revel/cmd/utils" "errors" - "fmt" "strings" "github.com/revel/cmd/logger" ) @@ -20,6 +18,7 @@ type ( log logger.MultiLogger packageList []*packages.Package importMap map[string]string + packageMap map[string]string sourceInfoProcessor *SourceInfoProcessor sourceInfo *model.SourceInfo } @@ -30,97 +29,10 @@ func ProcessSource(revelContainer *model.RevelContainer) (sourceInfo *model.Sour processor := NewSourceProcessor(revelContainer) compileError = processor.parse() sourceInfo = processor.sourceInfo - fmt.Printf("From parsers \n%v\n%v\n", sourceInfo, compileError) - //// Combine packages for modules and app and revel - //allPackages := []string{revelContainer.ImportPath+"/app/controllers/...",model.RevelImportPath} - //for _,module := range revelContainer.ModulePathMap { - // allPackages = append(allPackages,module.ImportPath+"/app/controllers/...") - //} - //allPackages = []string{revelContainer.ImportPath+"/app/controllers/..."} - // - //config := &packages.Config{ - // // ode: packages.NeedSyntax | packages.NeedCompiledGoFiles, - // Mode: packages.NeedTypes | packages.NeedSyntax , - // //Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | - // // packages.NeedImports | packages.NeedDeps | packages.NeedExportsFile | - // // packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo | - // // packages.NeedTypesSizes, - // - // //Mode: packages.NeedName | packages.NeedImports | packages.NeedDeps | packages.NeedExportsFile | packages.NeedFiles | - // // packages.NeedCompiledGoFiles | packages.NeedTypesSizes | - // // packages.NeedSyntax | packages.NeedCompiledGoFiles , - // //Mode: packages.NeedSyntax | packages.NeedCompiledGoFiles | packages.NeedName | packages.NeedFiles | - // // packages.LoadTypes | packages.NeedTypes | packages.NeedDeps, //, // | - // // packages.NeedTypes, // packages.LoadTypes | packages.NeedSyntax | packages.NeedTypesInfo, - // //packages.LoadSyntax | packages.NeedDeps, - // Dir:revelContainer.AppPath, - //} - //utils.Logger.Info("Before ","apppath", config.Dir,"paths",allPackages) - //pkgs, err := packages.Load(config, allPackages...) - //utils.Logger.Info("***Loaded packegs ", "len results", len(pkgs), "error",err) - //// Lets see if we can output all the path names - ////packages.Visit(pkgs,func(p *packages.Package) bool{ - //// println("Got pre",p.ID) - //// return true - ////}, func(p *packages.Package) { - ////}) - //counter := 0 - //for _, p := range pkgs { - // utils.Logger.Info("Errores","error",p.Errors, "id",p.ID) - // //for _,g := range p.GoFiles { - // // println("File", g) - // //} - // //for _, t:= range p.Syntax { - // // utils.Logger.Info("File","name",t.Name) - // //} - // //println("package typoe fouhnd ",p.Types.Name()) - // //imports := map[string]string{} - // - // for _,s := range p.Syntax { - // println("File ",s.Name.Name ) - // for _, decl := range s.Decls { - // genDecl, ok := decl.(*ast.GenDecl) - // if !ok { - // continue - // } - // - // if genDecl.Tok == token.IMPORT { - // for _, spec := range genDecl.Specs { - // importSpec := spec.(*ast.ImportSpec) - // fmt.Printf("*** import specification %#v\n", importSpec) - // var pkgAlias string - // if importSpec.Name != nil { - // pkgAlias = importSpec.Name.Name - // if pkgAlias == "_" { - // continue - // } - // } - // quotedPath := importSpec.Path.Value // e.g. "\"sample/app/models\"" - // fullPath := quotedPath[1 : len(quotedPath)-1] // Remove the quotes - // if pkgAlias == "" { - // pkgAlias = fullPath - // if index:=strings.LastIndex(pkgAlias,"/");index>0 { - // pkgAlias = pkgAlias[index+1:] - // } - // } - // //imports[pkgAlias] = fullPath - // println("Package ", pkgAlias, "fullpath", fullPath) - // } - // } - // } - // } - // } - // //p.Fset.Iterate(func(file *token.File) bool{ - // // - // // // utils.Logger.Info("Output","Found file", p.ID," AND NAME ", f.Name()) - // // // For each declaration in the source file... - // // //for _, decl := range file.Decls { - // // // addImports(imports, decl, pkgPath) - // // //} - // // counter ++ - // // return true - // //}) - ////} + if compileError==nil { + processor.log.Infof("From parsers : Structures:%d InitImports:%d ValidationKeys:%d %v", len(sourceInfo.StructSpecs), len(sourceInfo.InitImportPaths), len(sourceInfo.ValidationKeys),sourceInfo.PackageMap) + } + if false { compileError = errors.New("Incompleted") utils.Logger.Panic("Not implemented") @@ -143,15 +55,30 @@ func (s *SourceProcessor) parse() (compileError error) { if compileError = s.addSourceInfo(); compileError != nil { return } + s.sourceInfo.PackageMap = map[string]string{} + getImportFromMap := func(packagePath string) string { + for path := range s.packageMap { + if strings.Index(path,packagePath)==0 { + fullPath := s.packageMap[path] + return fullPath[:(len(fullPath) - len(path) + len(packagePath))] + } + } + return "" + } + s.sourceInfo.PackageMap[model.RevelImportPath] = getImportFromMap(model.RevelImportPath) + s.sourceInfo.PackageMap[s.revelContainer.ImportPath] = getImportFromMap(s.revelContainer.ImportPath) + for _, module := range s.revelContainer.ModulePathMap { + s.sourceInfo.PackageMap[module.ImportPath] = getImportFromMap(module.ImportPath) + } return } func (s *SourceProcessor) addPackages() (err error) { - allPackages := []string{s.revelContainer.ImportPath + "/..."} //,model.RevelImportPath} + allPackages := []string{s.revelContainer.ImportPath + "/...",model.RevelImportPath} for _, module := range s.revelContainer.ModulePathMap { allPackages = append(allPackages, module.ImportPath + "/...") // +"/app/controllers/...") } - allPackages = []string{s.revelContainer.ImportPath + "/..."} //+"/app/controllers/..."} + //allPackages = []string{s.revelContainer.ImportPath + "/..."} //+"/app/controllers/..."} config := &packages.Config{ // ode: packages.NeedSyntax | packages.NeedCompiledGoFiles, @@ -180,7 +107,9 @@ func (s *SourceProcessor) addPackages() (err error) { } func (s *SourceProcessor) addImportMap() (err error) { s.importMap = map[string]string{} + s.packageMap = map[string]string{} for _, p := range s.packageList { + if len(p.Errors) > 0 { // Generate a compile error for _, e := range p.Errors { diff --git a/revel/build.go b/revel/build.go index e929114..3bba537 100644 --- a/revel/build.go +++ b/revel/build.go @@ -37,6 +37,12 @@ func init() { // The update config updates the configuration command so that it can run func updateBuildConfig(c *model.CommandConfig, args []string) bool { c.Index = model.BUILD + if c.Build.TargetPath=="" { + c.Build.TargetPath="target" + } + if len(args)==0 && c.Build.ImportPath!="" { + return true + } // If arguments were passed in then there must be two if len(args) < 2 { fmt.Fprintf(os.Stderr, "%s\n%s", cmdBuild.UsageLine, cmdBuild.Long) @@ -63,7 +69,7 @@ func buildApp(c *model.CommandConfig) (err error) { c.Build.Mode = mode c.Build.ImportPath = appImportPath - revel_paths, err := model.NewRevelPaths(mode, appImportPath, "", model.NewWrappedRevelCallback(nil, c.PackageResolver)) + revel_paths, err := model.NewRevelPaths(mode, appImportPath, c.AppPath, model.NewWrappedRevelCallback(nil, c.PackageResolver)) if err != nil { return } @@ -88,7 +94,7 @@ func buildApp(c *model.CommandConfig) (err error) { if err != nil { return } - err = buildCopyModules(c, revel_paths, packageFolders) + err = buildCopyModules(c, revel_paths, packageFolders, app) if err != nil { return } @@ -148,7 +154,7 @@ func buildCopyFiles(c *model.CommandConfig, app *harness.App, revel_paths *model } // Based on the section copy over the build modules -func buildCopyModules(c *model.CommandConfig, revel_paths *model.RevelContainer, packageFolders []string) (err error) { +func buildCopyModules(c *model.CommandConfig, revel_paths *model.RevelContainer, packageFolders []string, app *harness.App) (err error) { destPath := filepath.Join(c.Build.TargetPath, "src") // Find all the modules used and copy them over. config := revel_paths.Config.Raw() @@ -174,14 +180,10 @@ func buildCopyModules(c *model.CommandConfig, revel_paths *model.RevelContainer, } } - modulePaths, err := utils.FindSrcPaths(c.AppPath, moduleImportList, c.PackageResolver) - - if err != nil { - utils.Logger.Fatalf("Failed to load modules ", "error", err) - } // Copy the the paths for each of the modules - for importPath, fsPath := range modulePaths { + for _,importPath := range moduleImportList { + fsPath := app.PackagePathMap[importPath] utils.Logger.Info("Copy files ", "to", filepath.Join(destPath, importPath), "from", fsPath) if c.Build.CopySource { err = utils.CopyDir(filepath.Join(destPath, importPath), fsPath, nil) diff --git a/revel/clean.go b/revel/clean.go index 6a210f3..6f428bd 100644 --- a/revel/clean.go +++ b/revel/clean.go @@ -37,6 +37,9 @@ func init() { // Update the clean command configuration, using old method func updateCleanConfig(c *model.CommandConfig, args []string) bool { c.Index = model.CLEAN + if len(args)==0 && c.Clean.ImportPath!="" { + return true + } if len(args) == 0 { fmt.Fprintf(os.Stderr, cmdClean.Long) return false diff --git a/revel/package.go b/revel/package.go index 8967258..723e7a9 100644 --- a/revel/package.go +++ b/revel/package.go @@ -40,6 +40,9 @@ func init() { // Called when unable to parse the command line automatically and assumes an old launch func updatePackageConfig(c *model.CommandConfig, args []string) bool { c.Index = model.PACKAGE + if len(args)==0 && c.Package.ImportPath!="" { + return true + } c.Package.ImportPath = args[0] if len(args) > 1 { c.Package.Mode = args[1] @@ -58,7 +61,7 @@ func packageApp(c *model.CommandConfig) (err error) { } appImportPath := c.ImportPath - revel_paths, err := model.NewRevelPaths(mode, appImportPath, "", model.NewWrappedRevelCallback(nil, c.PackageResolver)) + revel_paths, err := model.NewRevelPaths(mode, appImportPath, c.AppPath, model.NewWrappedRevelCallback(nil, c.PackageResolver)) if err != nil { return } diff --git a/revel/run.go b/revel/run.go index adce55c..85c9d3f 100644 --- a/revel/run.go +++ b/revel/run.go @@ -6,7 +6,7 @@ package main import ( "strconv" - + "encoding/json" "fmt" "github.com/revel/cmd/harness" "github.com/revel/cmd/model" @@ -159,7 +159,11 @@ func runApp(c *model.CommandConfig) (err error) { utils.Logger.Errorf("Failed to build app: %s", err) } app.Port = revel_path.HTTPPort - runMode := fmt.Sprintf(`{"mode":"%s", "specialUseFlag":%v}`, app.Paths.RunMode, c.Verbose) + var paths []byte + if len(app.PackagePathMap)>0 { + paths, _ = json.Marshal(app.PackagePathMap) + } + runMode := fmt.Sprintf(`{"mode":"%s", "specialUseFlag":%v,"packagePathMap":%s}`, app.Paths.RunMode, c.Verbose, string(paths)) if c.HistoricMode { runMode = revel_path.RunMode } diff --git a/revel/test.go b/revel/test.go index 8465f6b..abeb35b 100644 --- a/revel/test.go +++ b/revel/test.go @@ -55,6 +55,10 @@ func init() { // Called to update the config command with from the older stype func updateTestConfig(c *model.CommandConfig, args []string) bool { c.Index = model.TEST + if len(args)==0 && c.Test.ImportPath!="" { + return true + } + // The full test runs // revel test (run mode) (suite(.function)) if len(args) < 1 { @@ -78,7 +82,7 @@ func testApp(c *model.CommandConfig) (err error) { } // Find and parse app.conf - revel_path, err := model.NewRevelPaths(mode, c.ImportPath, "", model.NewWrappedRevelCallback(nil, c.PackageResolver)) + revel_path, err := model.NewRevelPaths(mode, c.ImportPath, c.AppPath, model.NewWrappedRevelCallback(nil, c.PackageResolver)) if err != nil { return } @@ -104,11 +108,16 @@ func testApp(c *model.CommandConfig) (err error) { if reverr != nil { return utils.NewBuildIfError(reverr, "Error building: ") } - runMode := fmt.Sprintf(`{"mode":"%s","testModeFlag":true, "specialUseFlag":%v}`, app.Paths.RunMode, c.Verbose) + var paths []byte + if len(app.PackagePathMap)>0 { + paths, _ = json.Marshal(app.PackagePathMap) + } + runMode := fmt.Sprintf(`{"mode":"%s", "specialUseFlag":%v,"packagePathMap":%s}`, app.Paths.RunMode, c.Verbose, string(paths)) if c.HistoricMode { runMode = app.Paths.RunMode } cmd := app.Cmd(runMode) + cmd.Dir=c.AppPath cmd.Stderr = io.MultiWriter(cmd.Stderr, file) cmd.Stdout = io.MultiWriter(cmd.Stderr, file) diff --git a/utils/file.go b/utils/file.go index 6272820..b48f9ff 100644 --- a/utils/file.go +++ b/utils/file.go @@ -4,8 +4,8 @@ import ( "archive/tar" "bytes" "compress/gzip" - "errors" "fmt" + "errors" "html/template" "io" "io/ioutil" @@ -352,7 +352,7 @@ func findSrcPaths(appPath string, packagesList []string) (sourcePathsmap map[str sourcePathsmap = map[string]string{} pkgs, err := packages.Load(config, packagesList...) - Logger.Info("Loaded packegs ", "len results", len(pkgs), "error",err) + Logger.Info("Loaded packegs ", "len results", len(pkgs), "error",err,"basedir",appPath) for _, packageName := range packagesList { found := false log:= Logger.New("seeking",packageName) @@ -360,11 +360,11 @@ func findSrcPaths(appPath string, packagesList []string) (sourcePathsmap map[str log.Info("Found package","package",pck.ID) if pck.ID == packageName { if pck.Errors!=nil && len(pck.Errors)>0 { - Logger.Info("Error ", "count", len(pck.Errors), "App Import Path", pck.ID,"errors",pck.Errors) + log.Info("Error ", "count", len(pck.Errors), "App Import Path", pck.ID,"errors",pck.Errors) } //a,_ := pck.MarshalJSON() - Logger.Info("Found ", "count", len(pck.GoFiles), "App Import Path", pck.ID) + log.Info("Found ", "count", len(pck.GoFiles), "App Import Path", pck.ID,"apppath",appPath) sourcePathsmap[packageName] = filepath.Dir(pck.GoFiles[0]) found = true }