Added manual scan on packages in app folder

This allows for source code generation. Packages in <application>/app folder are scanned manually as opposed to the `packages.Load` scan which will fast fail on compile error, and leave you with go files with no syntax.

.travis.yml

@@ -47,7 +47,7 @@ script:
   #- revel package my/testapp prod
 
   # Ensure the new-app flow works (plus the other commands).
-  - revel new     --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v
+  - revel new     --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 --package revelframework.com -v
   - revel test    --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v
   - revel clean   --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v
   - revel build   --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@develop" -a my/testapp2 -v -t build/testapp2

model/source_info.go

@@ -110,7 +110,7 @@ func (s *SourceInfo) TypesThatEmbed(targetType, packageFilter string) (filtered
 // ControllerSpecs returns the all the controllers that embeds
 // `revel.Controller`
 func (s *SourceInfo) ControllerSpecs() []*TypeInfo {
-	utils.Logger.Infof("Scanning controller specifications for types ","typePath",RevelImportPath + ".Controller", "speclen",len(s.controllerSpecs))
+	utils.Logger.Info("Scanning controller specifications for types ","typePath",RevelImportPath + ".Controller", "speclen",len(s.controllerSpecs))
 	if s.controllerSpecs == nil {
 		s.controllerSpecs = s.TypesThatEmbed(RevelImportPath + ".Controller", "controllers")
 	}

parser2/source_info_processor.go

@@ -34,6 +34,7 @@ func (s *SourceInfoProcessor) processPackage(p *packages.Package) (sourceInfo *m
 	)
 	localImportMap := map[string]string{}
 	log := s.sourceProcessor.log.New("package", p.PkgPath)
+	log.Info("Processing package")
 	for _, tree := range p.Syntax {
 		for _, decl := range tree.Decls {
 
@@ -41,8 +42,8 @@ func (s *SourceInfoProcessor) processPackage(p *packages.Package) (sourceInfo *m
 			if !s.addImport(decl, p, localImportMap, log) {
 				continue
 			}
-			// log.Info("*** checking", p.Fset.Position(decl.Pos()).Filename)
 			spec, found := s.getStructTypeDecl(decl, p.Fset)
+			//log.Info("Checking file","filename", p.Fset.Position(decl.Pos()).Filename,"found",found)
 			if found {
 				if isController || isTest {
 					controllerSpec := s.getControllerSpec(spec, p, localImportMap)

parser2/source_processor.go

@@ -4,10 +4,15 @@ import (
 	"github.com/revel/cmd/model"
 	"golang.org/x/tools/go/packages"
 	"github.com/revel/cmd/utils"
-	"errors"
+	"go/parser"
-
 	"strings"
 	"github.com/revel/cmd/logger"
+	"os"
+	"path/filepath"
+	"go/ast"
+	"go/token"
+	"go/scanner"
+
 )
 
 type (
@@ -31,10 +36,6 @@ func ProcessSource(revelContainer *model.RevelContainer) (sourceInfo *model.Sour
 		processor.log.Infof("From parsers : Structures:%d InitImports:%d ValidationKeys:%d %v", len(sourceInfo.StructSpecs), len(sourceInfo.InitImportPaths), len(sourceInfo.ValidationKeys), sourceInfo.PackageMap)
 	}
 
-	if false {
-		compileError = errors.New("Incompleted")
-		utils.Logger.Panic("Not implemented")
-	}
 	return
 }
 
@@ -54,6 +55,7 @@ func (s *SourceProcessor) parse() (compileError error) {
 	if compileError = s.addSourceInfo(); compileError != nil {
 		return
 	}
+
 	s.sourceInfo.PackageMap = map[string]string{}
 	getImportFromMap := func(packagePath string) string {
 		for path := range s.packageMap {
@@ -76,7 +78,7 @@ func (s *SourceProcessor) parse() (compileError error) {
 // Using the packages.Load function load all the packages and type specifications (forces compile).
 // this sets the SourceProcessor.packageList         []*packages.Package
 func (s *SourceProcessor) addPackages() (err error) {
-	allPackages := []string{s.revelContainer.ImportPath + "/...", model.RevelImportPath + "/..."}
+	allPackages := []string{model.RevelImportPath + "/..."}
 	for _, module := range s.revelContainer.ModulePathMap {
 		allPackages = append(allPackages, module.ImportPath + "/...") // +"/app/controllers/...")
 	}
@@ -105,10 +107,113 @@ func (s *SourceProcessor) addPackages() (err error) {
 		Dir:s.revelContainer.AppPath,
 	}
 	s.packageList, err = packages.Load(config, allPackages...)
-	s.log.Info("Loaded packages ", "len results", len(s.packageList), "error", err)
+	s.log.Info("Loaded modules ", "len results", len(s.packageList), "error", err)
+
+
+	// Now process the files in the aap source folder	s.revelContainer.ImportPath + "/...",
+	err = utils.Walk(s.revelContainer.AppPath, s.processPath)
+	s.log.Info("Loaded apps and modules ", "len results", len(s.packageList), "error", err)
 	return
 }
 
+// This callback is used to build the packages for the "app" package. This allows us to
+// parse the source files without doing a full compile on them
+// This callback only processes folders, so any files passed to this will return a nil
+func (s *SourceProcessor) processPath(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		s.log.Error("Error scanning app source:", "error", err)
+		return nil
+	}
+
+	// Ignore files and folders not marked tmp (since those are generated)
+	if !info.IsDir() || info.Name() == "tmp" {
+		return nil
+	}
+
+	// Real work for processing the folder
+	pkgImportPath := s.revelContainer.ImportPath
+	appPath := s.revelContainer.BasePath
+	if appPath != path {
+		pkgImportPath = s.revelContainer.ImportPath + "/" + filepath.ToSlash(path[len(appPath)+1:])
+	}
+	// Parse files within the path.
+	var pkgMap map[string]*ast.Package
+	fset := token.NewFileSet()
+	pkgMap, err = parser.ParseDir(
+		fset,
+		path,
+		func(f os.FileInfo) bool {
+			return !f.IsDir() && !strings.HasPrefix(f.Name(), ".") && strings.HasSuffix(f.Name(), ".go")
+		},
+		0)
+
+	if err != nil {
+		if errList, ok := err.(scanner.ErrorList); ok {
+			var pos = errList[0].Pos
+			newError := &utils.SourceError{
+				SourceType:  ".go source",
+				Title:       "Go Compilation Error",
+				Path:        pos.Filename,
+				Description: errList[0].Msg,
+				Line:        pos.Line,
+				Column:      pos.Column,
+				SourceLines: utils.MustReadLines(pos.Filename),
+			}
+
+			errorLink := s.revelContainer.Config.StringDefault("error.link", "")
+			if errorLink != "" {
+				newError.SetLink(errorLink)
+			}
+			return newError
+		}
+
+		// This is exception, err already checked above. Here just a print
+		ast.Print(nil, err)
+		s.log.Fatal("Failed to parse dir", "error", err)
+	}
+	// Skip "main" packages.
+	delete(pkgMap, "main")
+
+	// Ignore packages that end with _test
+	// These cannot be included in source code that is not generated specifically as a test
+	for i := range pkgMap {
+		if len(i) > 6 {
+			if string(i[len(i)-5:]) == "_test" {
+				delete(pkgMap, i)
+			}
+		}
+	}
+
+	// If there is no code in this directory, skip it.
+	if len(pkgMap) == 0 {
+		return nil
+	}
+
+	// There should be only one package in this directory.
+	if len(pkgMap) > 1 {
+		for i := range pkgMap {
+			println("Found package ", i)
+		}
+		utils.Logger.Fatal("Most unexpected! Multiple packages in a single directory:", "packages", pkgMap)
+	}
+
+	// At this point there is only one package in the pkgs map,
+	p := &packages.Package{}
+	p.PkgPath = pkgImportPath
+	p.Fset = fset
+	for _, pkg := range pkgMap {
+		p.Name = pkg.Name
+		s.log.Info("Found package","pkg.Name", pkg.Name,"p.Name", p.PkgPath)
+		for filename,astFile := range pkg.Files {
+			p.Syntax = append(p.Syntax,astFile)
+			p.GoFiles = append(p.GoFiles,filename)
+		}
+	}
+	s.packageList = append(s.packageList, p)
+
+	return nil
+}
+
 // This function is used to populate a map so that we can lookup controller embedded types in order to determine
 // if a Struct inherits from from revel.Controller
 func (s *SourceProcessor) addImportMap() (err error) {

revel/new.go

@@ -203,13 +203,13 @@ func setApplicationPath(c *model.CommandConfig) (err error) {
 	// revel/revel#1014 validate relative path, we cannot use built-in functions
 	// since Go import path is valid relative path too.
 	// so check basic part of the path, which is "."
-	if filepath.IsAbs(c.ImportPath) || strings.HasPrefix(c.ImportPath, ".") {
-		utils.Logger.Fatalf("Abort: '%s' looks like a directory.  Please provide a Go import path instead.",
-			c.ImportPath)
-	}
 
 	// If we are running a vendored version of Revel we do not need to check for it.
 	if !c.Vendored {
+		if filepath.IsAbs(c.ImportPath) || strings.HasPrefix(c.ImportPath, ".") {
+			utils.Logger.Fatalf("Abort: '%s' looks like a directory.  Please provide a Go import path instead.",
+				c.ImportPath)
+		}
 		_, err = build.Import(model.RevelImportPath, "", build.FindOnly)
 		if err != nil {
 			//// Go get the revel project

utils/file.go

@@ -203,7 +203,8 @@ func Walk(root string, walkFn filepath.WalkFunc) error {
 	return fsWalk(root, root, walkFn)
 }
 
-// Walk the tree using the function
+// Walk the path tree using the function
+// Every file found will call the function
 func fsWalk(fname string, linkName string, walkFn filepath.WalkFunc) error {
 	fsWalkFunc := func(path string, info os.FileInfo, err error) error {
 		if err != nil {