From 223bd3b7c073de4c9cd6068f8dbbe0929d7a22ac Mon Sep 17 00:00:00 2001 From: "notzippy@gmail.com" Date: Mon, 18 May 2020 11:47:01 -0700 Subject: [PATCH] Added manual scan on packages in app folder This allows for source code generation. Packages in /app folder are scanned manually as opposed to the `packages.Load` scan which will fast fail on compile error, and leave you with go files with no syntax. --- .travis.yml | 2 +- model/source_info.go | 2 +- parser2/source_info_processor.go | 3 +- parser2/source_processor.go | 121 +++++++++++++++++++++++++++++-- revel/new.go | 8 +- utils/file.go | 3 +- 6 files changed, 123 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 041e135..cd9a671 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ script: #- revel package my/testapp prod # Ensure the new-app flow works (plus the other commands). - - revel new --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v + - revel new --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 --package revelframework.com -v - revel test --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v - revel clean --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v - revel build --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v -t build/testapp2 diff --git a/model/source_info.go b/model/source_info.go index 2fbaaf5..8da1b26 100644 --- a/model/source_info.go +++ b/model/source_info.go @@ -110,7 +110,7 @@ func (s *SourceInfo) TypesThatEmbed(targetType, packageFilter string) (filtered // ControllerSpecs returns the all the controllers that embeds // `revel.Controller` func (s *SourceInfo) ControllerSpecs() []*TypeInfo { - utils.Logger.Infof("Scanning controller specifications for types ","typePath",RevelImportPath + ".Controller", "speclen",len(s.controllerSpecs)) + utils.Logger.Info("Scanning controller specifications for types ","typePath",RevelImportPath + ".Controller", "speclen",len(s.controllerSpecs)) if s.controllerSpecs == nil { s.controllerSpecs = s.TypesThatEmbed(RevelImportPath + ".Controller", "controllers") } diff --git a/parser2/source_info_processor.go b/parser2/source_info_processor.go index f6d98de..40d3f85 100644 --- a/parser2/source_info_processor.go +++ b/parser2/source_info_processor.go @@ -34,6 +34,7 @@ func (s *SourceInfoProcessor) processPackage(p *packages.Package) (sourceInfo *m ) localImportMap := map[string]string{} log := s.sourceProcessor.log.New("package", p.PkgPath) + log.Info("Processing package") for _, tree := range p.Syntax { for _, decl := range tree.Decls { @@ -41,8 +42,8 @@ func (s *SourceInfoProcessor) processPackage(p *packages.Package) (sourceInfo *m if !s.addImport(decl, p, localImportMap, log) { continue } - // log.Info("*** checking", p.Fset.Position(decl.Pos()).Filename) spec, found := s.getStructTypeDecl(decl, p.Fset) + //log.Info("Checking file","filename", p.Fset.Position(decl.Pos()).Filename,"found",found) if found { if isController || isTest { controllerSpec := s.getControllerSpec(spec, p, localImportMap) diff --git a/parser2/source_processor.go b/parser2/source_processor.go index 89443b5..dae3f39 100644 --- a/parser2/source_processor.go +++ b/parser2/source_processor.go @@ -4,10 +4,15 @@ import ( "github.com/revel/cmd/model" "golang.org/x/tools/go/packages" "github.com/revel/cmd/utils" - "errors" - + "go/parser" "strings" "github.com/revel/cmd/logger" + "os" + "path/filepath" + "go/ast" + "go/token" + "go/scanner" + ) type ( @@ -31,10 +36,6 @@ func ProcessSource(revelContainer *model.RevelContainer) (sourceInfo *model.Sour processor.log.Infof("From parsers : Structures:%d InitImports:%d ValidationKeys:%d %v", len(sourceInfo.StructSpecs), len(sourceInfo.InitImportPaths), len(sourceInfo.ValidationKeys), sourceInfo.PackageMap) } - if false { - compileError = errors.New("Incompleted") - utils.Logger.Panic("Not implemented") - } return } @@ -54,6 +55,7 @@ func (s *SourceProcessor) parse() (compileError error) { if compileError = s.addSourceInfo(); compileError != nil { return } + s.sourceInfo.PackageMap = map[string]string{} getImportFromMap := func(packagePath string) string { for path := range s.packageMap { @@ -76,7 +78,7 @@ func (s *SourceProcessor) parse() (compileError error) { // Using the packages.Load function load all the packages and type specifications (forces compile). // this sets the SourceProcessor.packageList []*packages.Package func (s *SourceProcessor) addPackages() (err error) { - allPackages := []string{s.revelContainer.ImportPath + "/...", model.RevelImportPath + "/..."} + allPackages := []string{model.RevelImportPath + "/..."} for _, module := range s.revelContainer.ModulePathMap { allPackages = append(allPackages, module.ImportPath + "/...") // +"/app/controllers/...") } @@ -105,10 +107,113 @@ func (s *SourceProcessor) addPackages() (err error) { Dir:s.revelContainer.AppPath, } s.packageList, err = packages.Load(config, allPackages...) - s.log.Info("Loaded packages ", "len results", len(s.packageList), "error", err) + s.log.Info("Loaded modules ", "len results", len(s.packageList), "error", err) + + + // Now process the files in the aap source folder s.revelContainer.ImportPath + "/...", + err = utils.Walk(s.revelContainer.AppPath, s.processPath) + s.log.Info("Loaded apps and modules ", "len results", len(s.packageList), "error", err) return } +// This callback is used to build the packages for the "app" package. This allows us to +// parse the source files without doing a full compile on them +// This callback only processes folders, so any files passed to this will return a nil +func (s *SourceProcessor) processPath(path string, info os.FileInfo, err error) error { + if err != nil { + s.log.Error("Error scanning app source:", "error", err) + return nil + } + + // Ignore files and folders not marked tmp (since those are generated) + if !info.IsDir() || info.Name() == "tmp" { + return nil + } + + // Real work for processing the folder + pkgImportPath := s.revelContainer.ImportPath + appPath := s.revelContainer.BasePath + if appPath != path { + pkgImportPath = s.revelContainer.ImportPath + "/" + filepath.ToSlash(path[len(appPath)+1:]) + } + // Parse files within the path. + var pkgMap map[string]*ast.Package + fset := token.NewFileSet() + pkgMap, err = parser.ParseDir( + fset, + path, + func(f os.FileInfo) bool { + return !f.IsDir() && !strings.HasPrefix(f.Name(), ".") && strings.HasSuffix(f.Name(), ".go") + }, + 0) + + if err != nil { + if errList, ok := err.(scanner.ErrorList); ok { + var pos = errList[0].Pos + newError := &utils.SourceError{ + SourceType: ".go source", + Title: "Go Compilation Error", + Path: pos.Filename, + Description: errList[0].Msg, + Line: pos.Line, + Column: pos.Column, + SourceLines: utils.MustReadLines(pos.Filename), + } + + errorLink := s.revelContainer.Config.StringDefault("error.link", "") + if errorLink != "" { + newError.SetLink(errorLink) + } + return newError + } + + // This is exception, err already checked above. Here just a print + ast.Print(nil, err) + s.log.Fatal("Failed to parse dir", "error", err) + } + // Skip "main" packages. + delete(pkgMap, "main") + + // Ignore packages that end with _test + // These cannot be included in source code that is not generated specifically as a test + for i := range pkgMap { + if len(i) > 6 { + if string(i[len(i)-5:]) == "_test" { + delete(pkgMap, i) + } + } + } + + // If there is no code in this directory, skip it. + if len(pkgMap) == 0 { + return nil + } + + // There should be only one package in this directory. + if len(pkgMap) > 1 { + for i := range pkgMap { + println("Found package ", i) + } + utils.Logger.Fatal("Most unexpected! Multiple packages in a single directory:", "packages", pkgMap) + } + + // At this point there is only one package in the pkgs map, + p := &packages.Package{} + p.PkgPath = pkgImportPath + p.Fset = fset + for _, pkg := range pkgMap { + p.Name = pkg.Name + s.log.Info("Found package","pkg.Name", pkg.Name,"p.Name", p.PkgPath) + for filename,astFile := range pkg.Files { + p.Syntax = append(p.Syntax,astFile) + p.GoFiles = append(p.GoFiles,filename) + } + } + s.packageList = append(s.packageList, p) + + return nil +} + // This function is used to populate a map so that we can lookup controller embedded types in order to determine // if a Struct inherits from from revel.Controller func (s *SourceProcessor) addImportMap() (err error) { diff --git a/revel/new.go b/revel/new.go index 147d08a..f8cf922 100644 --- a/revel/new.go +++ b/revel/new.go @@ -203,13 +203,13 @@ func setApplicationPath(c *model.CommandConfig) (err error) { // revel/revel#1014 validate relative path, we cannot use built-in functions // since Go import path is valid relative path too. // so check basic part of the path, which is "." - if filepath.IsAbs(c.ImportPath) || strings.HasPrefix(c.ImportPath, ".") { - utils.Logger.Fatalf("Abort: '%s' looks like a directory. Please provide a Go import path instead.", - c.ImportPath) - } // If we are running a vendored version of Revel we do not need to check for it. if !c.Vendored { + if filepath.IsAbs(c.ImportPath) || strings.HasPrefix(c.ImportPath, ".") { + utils.Logger.Fatalf("Abort: '%s' looks like a directory. Please provide a Go import path instead.", + c.ImportPath) + } _, err = build.Import(model.RevelImportPath, "", build.FindOnly) if err != nil { //// Go get the revel project diff --git a/utils/file.go b/utils/file.go index 9a7ff02..0aafb70 100644 --- a/utils/file.go +++ b/utils/file.go @@ -203,7 +203,8 @@ func Walk(root string, walkFn filepath.WalkFunc) error { return fsWalk(root, root, walkFn) } -// Walk the tree using the function +// Walk the path tree using the function +// Every file found will call the function func fsWalk(fname string, linkName string, walkFn filepath.WalkFunc) error { fsWalkFunc := func(path string, info os.FileInfo, err error) error { if err != nil {