revel-cmd/harness/build.go

// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package harness

import (
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/revel/cmd/model"
	"github.com/revel/cmd/parser"
	"github.com/revel/cmd/utils"
)

var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"")

type ByString []*model.TypeInfo

func (c ByString) Len() int           { return len(c) }
func (c ByString) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
func (c ByString) Less(i, j int) bool { return c[i].String() < c[j].String() }

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build(c *model.CommandConfig, paths *model.RevelContainer, buildFlags ...string) (app *App, compileError *utils.Error) {
	// First, clear the generated files (to avoid them messing with ProcessSource).
	cleanSource(paths, "tmp", "routes")

	sourceInfo, compileError := parser.ProcessSource(paths)
	if compileError != nil {
		return
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := paths.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, strings.Split(dbImportPath, ",")...)
	}

	// Sort controllers so that file generation is reproducible
	controllers := sourceInfo.ControllerSpecs()
	sort.Stable(ByString(controllers))

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    controllers,
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource(paths, "tmp", "main.go", RevelMainTemplate, templateArgs)
	genSource(paths, "routes", "routes.go", RevelRoutesTemplate, templateArgs)

	// Read build config.
	buildTags := paths.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		utils.Logger.Fatal("Go executable not found in PATH.")
	}

	// Detect if deps tool should be used (is there a vendor folder ?)
	useVendor := utils.DirExists(filepath.Join(paths.BasePath, "vendor"))
	basePath := paths.BasePath
	for !useVendor {
		basePath = filepath.Dir(basePath)
		found := false
		// Check to see if we are still in the GOPATH
		for _, gopath := range filepath.SplitList(build.Default.GOPATH) {
			if strings.HasPrefix(basePath, gopath) {
				found = true
				break
			}
		}
		if !found {
			break
		} else {
			useVendor = utils.DirExists(filepath.Join(basePath, "vendor"))
		}
	}

	var depPath string
	if useVendor {
		utils.Logger.Info("Vendor folder detected, scanning for deps in path")
		depPath, err = exec.LookPath("dep")
		if err != nil {
			// Do not halt build unless a new package needs to be imported
			utils.Logger.Warn("Build: `dep` executable not found in PATH, but vendor folder detected." +
				"Packages can only be added automatically to the vendor folder using the `dep` tool. " +
				"You can install the `dep` tool by doing a `go get -u github.com/golang/dep/cmd/dep`")
		}
	} else {
		utils.Logger.Info("No vendor folder detected, not using dependency manager to import files")
	}

	pkg, err := build.Default.Import(paths.ImportPath, "", build.FindOnly)
	if err != nil {
		utils.Logger.Fatal("Failure importing", "path", paths.ImportPath)
	}

	// Binary path is a combination of $GOBIN/revel.d directory, app's import path and its name.
	binName := filepath.Join(pkg.BinDir, "revel.d", paths.ImportPath, filepath.Base(paths.BasePath))

	// Change binary path for Windows build
	goos := runtime.GOOS
	if goosEnv := os.Getenv("GOOS"); goosEnv != "" {
		goos = goosEnv
	}
	if goos == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	contains := func (s []string, e string) bool {
				for _, a := range s {
					if a == e {
					return true
				}
			}
		return false
		}

	for {
		appVersion := getAppVersion(paths)

		buildTime := time.Now().UTC().Format(time.RFC3339)
		versionLinkerFlags := fmt.Sprintf("-X %s/app.AppVersion=%s -X %s/app.BuildTime=%s",
			paths.ImportPath, appVersion, paths.ImportPath, buildTime)

		// Append any build flags specified, they will override existing flags
		flags := []string{}
		if len(c.BuildFlags)==0 {
			flags = []string{
				"build",
				"-i",
				"-ldflags", versionLinkerFlags,
				"-tags", buildTags,
				"-o", binName}
		} else {
			if !contains(c.BuildFlags,"build") {
				flags = []string{"build"}
			}
			flags = append(flags,c.BuildFlags...)
			if !contains(flags, "-ldflags") {
				flags = append(flags,"-ldflags", versionLinkerFlags)
			}
			if !contains(flags, "-tags") {
				flags = append(flags,"-tags", buildTags)
			}
			if !contains(flags, "-o") {
				flags = append(flags,"-o", binName)
			}
		}


		// Add in build flags
		flags = append(flags, buildFlags...)

		// This is Go main path
		gopath := c.GoPath
		for _, o := range paths.ModulePathMap {
			gopath += string(filepath.ListSeparator) + o
		}

		// Note: It's not applicable for filepath.* usage
		flags = append(flags, path.Join(paths.ImportPath, "app", "tmp"))

		buildCmd := exec.Command(goPath, flags...)
		buildCmd.Env = append(os.Environ(),
			"GOPATH="+gopath,
		)
		utils.Logger.Info("Exec:", "args", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName, paths), nil
		}

		// Since there was an error, capture the output in case we need to report it
		stOutput := string(output)

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindAllStringSubmatch(stOutput, -1)
		utils.Logger.Info("Build failed checking for missing imports", "message", stOutput, "missing_imports", len(matches))
		if matches == nil {
			utils.Logger.Info("Build failed no missing imports", "message", stOutput)
			return nil, newCompileError(paths, output)
		}
		utils.Logger.Warn("Detected missing packages, importing them", "packages", len(matches))
		for _, match := range matches {
			// Ensure we haven't already tried to go get it.
			pkgName := match[1]
			utils.Logger.Info("Trying to import ", "package", pkgName)
			if _, alreadyTried := gotten[pkgName]; alreadyTried {
				utils.Logger.Error("Failed to import ", "package", pkgName)
				return nil, newCompileError(paths, output)
			}
			gotten[pkgName] = struct{}{}

			// Execute "go get <pkg>"
			// Or dep `dep ensure -add <pkg>` if it is there
			var getCmd *exec.Cmd
			if useVendor {
				if depPath == "" {
					utils.Logger.Warn("Build: Vendor folder found, but the `dep` tool was not found, " +
						"if you use a different vendoring (package management) tool please add the following packages by hand, " +
						"or install the `dep` tool into your gopath by doing a `go get -u github.com/golang/dep/cmd/dep`. " +
						"For more information and usage of the tool please see http://github.com/golang/dep")
					for _, pkg := range matches {
						utils.Logger.Warn("Missing package", "package", pkg[1])
					}
				}
				getCmd = exec.Command(depPath, "ensure", "-add", pkgName)
				getCmd.Dir = paths.AppPath

			} else {
				getCmd = exec.Command(goPath, "get", pkgName)
			}
			utils.Logger.Info("Exec:", "args", getCmd.Args)
			getOutput, err := getCmd.CombinedOutput()
			if err != nil {
				utils.Logger.Error("Build failed", "message", stOutput)
				utils.Logger.Error("Failed to fetch the output", string(getOutput))
				return nil, newCompileError(paths, output)
			}
		}

		// Success getting the import, attempt to build again.
	}

	// TODO remove this unreachable code and document it
	utils.Logger.Fatal("Not reachable")
	return nil, nil
}

// Try to define a version string for the compiled app
// The following is tried (first match returns):
// - Read a version explicitly specified in the APP_VERSION environment
//   variable
// - Read the output of "git describe" if the source is in a git repository
// If no version can be determined, an empty string is returned.
func getAppVersion(paths *model.RevelContainer) string {
	if version := os.Getenv("APP_VERSION"); version != "" {
		return version
	}

	// Check for the git binary
	if gitPath, err := exec.LookPath("git"); err == nil {
		// Check for the .git directory
		gitDir := filepath.Join(paths.BasePath, ".git")
		info, err := os.Stat(gitDir)
		if (err != nil && os.IsNotExist(err)) || !info.IsDir() {
			return ""
		}
		gitCmd := exec.Command(gitPath, "--git-dir="+gitDir, "--work-tree="+paths.BasePath, "describe", "--always", "--dirty")
		utils.Logger.Info("Exec:", "args", gitCmd.Args)
		output, err := gitCmd.Output()

		if err != nil {
			utils.Logger.Error("Cannot determine git repository version:", "error", err)
			return ""
		}

		return "git-" + strings.TrimSpace(string(output))
	}

	return ""
}

func cleanSource(paths *model.RevelContainer, dirs ...string) {
	for _, dir := range dirs {
		cleanDir(paths, dir)
	}
}

func cleanDir(paths *model.RevelContainer, dir string) {
	utils.Logger.Info("Cleaning dir ", "dir", dir)
	tmpPath := filepath.Join(paths.AppPath, dir)
	f, err := os.Open(tmpPath)
	if err != nil {
		if !os.IsNotExist(err) {
			utils.Logger.Error("Failed to clean dir:", "error", err)
		}
	} else {
		defer func() {
			_ = f.Close()
		}()

		infos, err := f.Readdir(0)
		if err != nil {
			if !os.IsNotExist(err) {
				utils.Logger.Fatal("Failed to clean dir:", "error", err)
			}
		} else {
			for _, info := range infos {
				pathName := filepath.Join(tmpPath, info.Name())
				if info.IsDir() {
					err := os.RemoveAll(pathName)
					if err != nil {
						utils.Logger.Fatal("Failed to remove dir:", "error", err)
					}
				} else {
					err := os.Remove(pathName)
					if err != nil {
						utils.Logger.Fatal("Failed to remove file:", "error", err)
					}
				}
			}
		}
	}
}

// genSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(paths *model.RevelContainer, dir, filename, templateSource string, args map[string]interface{}) {
	cleanSource(paths, dir)

	err := utils.MustGenerateTemplate(filepath.Join(paths.AppPath, dir, filename), templateSource, args)
	if err != nil {
		utils.Logger.Fatal("Failed to generate template for source file", "error", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *model.SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*model.TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

// Adds an alias to the map of alias names
func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

// Generates a package alias
func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) || alias == "revel" {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

// Returns true if this value is in the map
func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(paths *model.RevelContainer, output []byte) *utils.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		errorMatch = regexp.MustCompile(`(?m)^(.*?):(\d+):\s(.*?)$`).FindSubmatch(output)

		if errorMatch == nil {
			utils.Logger.Error("Failed to parse build errors", "error", string(output))
			return &utils.Error{
				SourceType:  "Go code",
				Title:       "Go Compilation Error",
				Description: "See console for build error.",
			}
		}

		errorMatch = append(errorMatch, errorMatch[3])

		utils.Logger.Error("Build errors", "errors", string(output))
	}

	findInPaths := func(relFilename string) string {
		// Extract the paths from the gopaths, and search for file there first
		gopaths := filepath.SplitList(build.Default.GOPATH)
		for _, gp := range gopaths {
			newPath := filepath.Join(gp, relFilename)
			if utils.Exists(newPath) {
				return newPath
			}
		}
		newPath, _ := filepath.Abs(relFilename)
		utils.Logger.Warn("Could not find in GO path", "file", relFilename)
		return newPath
	}

	// Read the source for the offending file.
	var (
		relFilename  = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename  = findInPaths(relFilename)
		line, _      = strconv.Atoi(string(errorMatch[2]))
		description  = string(errorMatch[4])
		compileError = &utils.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	errorLink := paths.Config.StringDefault("error.link", "")

	if errorLink != "" {
		compileError.SetLink(errorLink)
	}

	fileStr, err := utils.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		utils.Logger.Error("Unable to readlines "+compileError.MetaError, "error", err)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

// RevelMainTemplate template for app/tmp/main.go
const RevelMainTemplate = `// GENERATED CODE - DO NOT EDIT
// This file is the main file for Revel.
// It registers all the controllers and provides details for the Revel server engine to
// properly inject parameters directly into the action endpoints.
package main

import (
	"flag"
	"reflect"
	"github.com/revel/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
	"github.com/revel/revel/testing"
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.AppLog.Info("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	testing.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`

// RevelRoutesTemplate template for app/conf/routes
const RevelRoutesTemplate = `// GENERATED CODE - DO NOT EDIT
// This file provides a way of creating URL's based on all the actions
// found in all the controllers.
package routes

import "github.com/revel/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (_ t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).URL
}
{{end}}
{{end}}
`