tinygo/loader/loader.go

package loader

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/scanner"
	"go/token"
	"go/types"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"text/template"

	"github.com/tinygo-org/tinygo/cgo"
	"github.com/tinygo-org/tinygo/goenv"
	"golang.org/x/tools/go/packages"
)

// Program holds all packages and some metadata about the program as a whole.
type Program struct {
	Build        *build.Context
	Tests        bool
	Packages     map[string]*Package
	MainPkg      *Package
	sorted       []*Package
	fset         *token.FileSet
	TypeChecker  types.Config
	Dir          string // current working directory (for error reporting)
	TINYGOROOT   string // root of the TinyGo installation or root of the source code
	CFlags       []string
	LDFlags      []string
	ClangHeaders string
}

// Package holds a loaded package, its imports, and its parsed files.
type Package struct {
	*Program
	*packages.Package
	Files []*ast.File
	Pkg   *types.Package
	types.Info
}

// Load loads the given package with all dependencies (including the runtime
// package). Call .Parse() afterwards to parse all Go files (including CGo
// processing, if necessary).
func (p *Program) Load(importPath string) error {
	if p.Packages == nil {
		p.Packages = make(map[string]*Package)
	}

	err := p.loadPackage(importPath)
	if err != nil {
		return err
	}
	p.MainPkg = p.sorted[len(p.sorted)-1]
	if _, ok := p.Packages["runtime"]; !ok {
		// The runtime package wasn't loaded. Although `go list -deps` seems to
		// return the full dependency list, there is no way to get those
		// packages from the go/packages package. Therefore load the runtime
		// manually and add it to the list of to-be-compiled packages
		// (duplicates are already filtered).
		return p.loadPackage("runtime")
	}
	return nil
}

func (p *Program) loadPackage(importPath string) error {
	cgoEnabled := "0"
	if p.Build.CgoEnabled {
		cgoEnabled = "1"
	}
	pkgs, err := packages.Load(&packages.Config{
		Mode:       packages.NeedName | packages.NeedFiles | packages.NeedImports | packages.NeedDeps,
		Env:        append(os.Environ(), "GOROOT="+p.Build.GOROOT, "GOOS="+p.Build.GOOS, "GOARCH="+p.Build.GOARCH, "CGO_ENABLED="+cgoEnabled),
		BuildFlags: []string{"-tags", strings.Join(p.Build.BuildTags, " ")},
		Tests:      p.Tests,
	}, importPath)
	if err != nil {
		return err
	}
	var pkg *packages.Package
	if p.Tests {
		// We need the second package. Quoting from the docs:
		// > For example, when using the go command, loading "fmt" with Tests=true
		// > returns four packages, with IDs "fmt" (the standard package),
		// > "fmt [fmt.test]" (the package as compiled for the test),
		// > "fmt_test" (the test functions from source files in package fmt_test),
		// > and "fmt.test" (the test binary).
		pkg = pkgs[1]
	} else {
		if len(pkgs) != 1 {
			return fmt.Errorf("expected exactly one package while importing %s, got %d", importPath, len(pkgs))
		}
		pkg = pkgs[0]
	}
	var importError *Errors
	var addPackages func(pkg *packages.Package)
	addPackages = func(pkg *packages.Package) {
		if _, ok := p.Packages[pkg.PkgPath]; ok {
			return
		}
		pkg2 := p.newPackage(pkg)
		p.Packages[pkg.PkgPath] = pkg2
		if len(pkg.Errors) != 0 {
			if importError != nil {
				// There was another error reported already. Do not report
				// errors from multiple packages at once.
				return
			}
			importError = &Errors{
				Pkg: pkg2,
			}
			for _, err := range pkg.Errors {
				pos := token.Position{}
				fields := strings.Split(err.Pos, ":")
				if len(fields) >= 2 {
					// There is some file/line/column information.
					if n, err := strconv.Atoi(fields[len(fields)-2]); err == nil {
						// Format: filename.go:line:colum
						pos.Filename = strings.Join(fields[:len(fields)-2], ":")
						pos.Line = n
						pos.Column, _ = strconv.Atoi(fields[len(fields)-1])
					} else {
						// Format: filename.go:line
						pos.Filename = strings.Join(fields[:len(fields)-1], ":")
						pos.Line, _ = strconv.Atoi(fields[len(fields)-1])
					}
					pos.Filename = p.getOriginalPath(pos.Filename)
				}
				importError.Errs = append(importError.Errs, scanner.Error{
					Pos: pos,
					Msg: err.Msg,
				})
			}
			return
		}

		// Get the list of imports (sorted alphabetically).
		names := make([]string, 0, len(pkg.Imports))
		for name := range pkg.Imports {
			names = append(names, name)
		}
		sort.Strings(names)

		// Add all the imports.
		for _, name := range names {
			addPackages(pkg.Imports[name])
		}

		p.sorted = append(p.sorted, pkg2)
	}
	addPackages(pkg)
	if importError != nil {
		return *importError
	}
	return nil
}

// getOriginalPath looks whether this path is in the generated GOROOT and if so,
// replaces the path with the original path (in GOROOT or TINYGOROOT). Otherwise
// the input path is returned.
func (p *Program) getOriginalPath(path string) string {
	originalPath := path
	if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) {
		// If this file is part of the synthetic GOROOT, try to infer the
		// original path.
		relpath := path[len(filepath.Join(p.Build.GOROOT, "src"))+1:]
		realgorootPath := filepath.Join(goenv.Get("GOROOT"), "src", relpath)
		if _, err := os.Stat(realgorootPath); err == nil {
			originalPath = realgorootPath
		}
		maybeInTinyGoRoot := false
		for prefix := range pathsToOverride(needsSyscallPackage(p.Build.BuildTags)) {
			if !strings.HasPrefix(relpath, prefix) {
				continue
			}
			maybeInTinyGoRoot = true
		}
		if maybeInTinyGoRoot {
			tinygoPath := filepath.Join(p.TINYGOROOT, "src", relpath)
			if _, err := os.Stat(tinygoPath); err == nil {
				originalPath = tinygoPath
			}
		}
	}
	return originalPath
}

// newPackage instantiates a new *Package object with initialized members.
func (p *Program) newPackage(pkg *packages.Package) *Package {
	return &Package{
		Program: p,
		Package: pkg,
		Info: types.Info{
			Types:      make(map[ast.Expr]types.TypeAndValue),
			Defs:       make(map[*ast.Ident]types.Object),
			Uses:       make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Scopes:     make(map[ast.Node]*types.Scope),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
		},
	}
}

// Sorted returns a list of all packages, sorted in a way that no packages come
// before the packages they depend upon.
func (p *Program) Sorted() []*Package {
	return p.sorted
}

// Parse parses all packages and typechecks them.
//
// The returned error may be an Errors error, which contains a list of errors.
//
// Idempotent.
func (p *Program) Parse() error {
	// Parse all packages.
	for _, pkg := range p.Sorted() {
		err := pkg.Parse()
		if err != nil {
			return err
		}
	}

	if p.Tests {
		err := p.swapTestMain()
		if err != nil {
			return err
		}
	}

	// Typecheck all packages.
	for _, pkg := range p.Sorted() {
		err := pkg.Check()
		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Program) swapTestMain() error {
	var tests []string

	isTestFunc := func(f *ast.FuncDecl) bool {
		// TODO: improve signature check
		if strings.HasPrefix(f.Name.Name, "Test") && f.Name.Name != "TestMain" {
			return true
		}
		return false
	}
	for _, f := range p.MainPkg.Files {
		for i, d := range f.Decls {
			switch v := d.(type) {
			case *ast.FuncDecl:
				if isTestFunc(v) {
					tests = append(tests, v.Name.Name)
				}
				if v.Name.Name == "main" {
					// Remove main
					if len(f.Decls) == 1 {
						f.Decls = make([]ast.Decl, 0)
					} else {
						f.Decls[i] = f.Decls[len(f.Decls)-1]
						f.Decls = f.Decls[:len(f.Decls)-1]
					}
				}
			}
		}
	}

	// TODO: Check if they defined a TestMain and call it instead of testing.TestMain
	const mainBody = `package main

import (
	"testing"
)

func main () {
	m := &testing.M{
		Tests: []testing.TestToCall{
{{range .TestFunctions}}
			{Name: "{{.}}", Func: {{.}}},
{{end}}
		},
	}

	testing.TestMain(m)
}
`
	tmpl := template.Must(template.New("testmain").Parse(mainBody))
	b := bytes.Buffer{}
	tmplData := struct {
		TestFunctions []string
	}{
		TestFunctions: tests,
	}

	err := tmpl.Execute(&b, tmplData)
	if err != nil {
		return err
	}
	path := filepath.Join(p.MainPkg.Dir, "$testmain.go")

	if p.fset == nil {
		p.fset = token.NewFileSet()
	}

	newMain, err := parser.ParseFile(p.fset, path, b.Bytes(), parser.AllErrors)
	if err != nil {
		return err
	}
	p.MainPkg.Files = append(p.MainPkg.Files, newMain)

	return nil
}

// parseFile is a wrapper around parser.ParseFile.
func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
	if p.fset == nil {
		p.fset = token.NewFileSet()
	}

	rd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer rd.Close()
	return parser.ParseFile(p.fset, p.getOriginalPath(path), rd, mode)
}

// Parse parses and typechecks this package.
//
// Idempotent.
func (p *Package) Parse() error {
	if len(p.Files) != 0 {
		return nil
	}

	// Load the AST.
	// TODO: do this in parallel.
	if p.PkgPath == "unsafe" {
		// Special case for the unsafe package. Don't even bother loading
		// the files.
		p.Pkg = types.Unsafe
		return nil
	}

	files, err := p.parseFiles()
	if err != nil {
		return err
	}
	p.Files = files

	return nil
}

// Check runs the package through the typechecker. The package must already be
// loaded and all dependencies must have been checked already.
//
// Idempotent.
func (p *Package) Check() error {
	if p.Pkg != nil {
		return nil
	}

	var typeErrors []error
	checker := p.TypeChecker
	checker.Error = func(err error) {
		typeErrors = append(typeErrors, err)
	}

	// Do typechecking of the package.
	checker.Importer = p

	typesPkg, err := checker.Check(p.PkgPath, p.fset, p.Files, &p.Info)
	if err != nil {
		if err, ok := err.(Errors); ok {
			return err
		}
		return Errors{p, typeErrors}
	}
	p.Pkg = typesPkg
	return nil
}

// parseFiles parses the loaded list of files and returns this list.
func (p *Package) parseFiles() ([]*ast.File, error) {
	// TODO: do this concurrently.
	var files []*ast.File
	var fileErrs []error

	var cgoFiles []*ast.File
	for _, file := range p.GoFiles {
		f, err := p.parseFile(file, parser.ParseComments)
		if err != nil {
			fileErrs = append(fileErrs, err)
			continue
		}
		if err != nil {
			fileErrs = append(fileErrs, err)
			continue
		}
		for _, importSpec := range f.Imports {
			if importSpec.Path.Value == `"C"` {
				cgoFiles = append(cgoFiles, f)
			}
		}
		files = append(files, f)
	}
	if len(cgoFiles) != 0 {
		cflags := append(p.CFlags, "-I"+filepath.Dir(p.GoFiles[0]))
		if p.ClangHeaders != "" {
			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.ClangHeaders)
		}
		generated, ldflags, errs := cgo.Process(files, p.Program.Dir, p.fset, cflags)
		if errs != nil {
			fileErrs = append(fileErrs, errs...)
		}
		files = append(files, generated)
		p.LDFlags = append(p.LDFlags, ldflags...)
	}
	if len(fileErrs) != 0 {
		return nil, Errors{p, fileErrs}
	}

	return files, nil
}

// Import implements types.Importer. It loads and parses packages it encounters
// along the way, if needed.
func (p *Package) Import(to string) (*types.Package, error) {
	if to == "unsafe" {
		return types.Unsafe, nil
	}
	if _, ok := p.Imports[to]; ok {
		return p.Packages[p.Imports[to].PkgPath].Pkg, nil
	} else {
		return nil, errors.New("package not imported: " + to)
	}
}