@ -2,165 +2,178 @@ package loader
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"go/ast"
"go/build"
"go/parser"
"go/scanner"
"go/token"
"go/types"
"io"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"text/template "
"syscall "
"github.com/tinygo-org/tinygo/cgo"
"github.com/tinygo-org/tinygo/compileopts"
"github.com/tinygo-org/tinygo/goenv"
"golang.org/x/tools/go/packages"
)
// Program holds all packages and some metadata about the program as a whole.
type Program struct {
Build * build . Context
Tests bool
Packages map [ string ] * Package
MainPkg * Package
sorted [ ] * Package
fset * token . FileSet
TypeChecker types . Config
Dir string // current working directory (for error reporting)
TINYGOROOT string // root of the TinyGo installation or root of the source code
CFlags [ ] string
LDFlags [ ] string
ClangHeaders string
config * compileopts . Config
clangHeaders string
typeChecker types . Config
goroot string // synthetic GOROOT
workingDir string
Packages map [ string ] * Package
sorted [ ] * Package
fset * token . FileSet
// Information obtained during parsing.
LDFlags [ ] string
}
// PackageJSON is a subset of the JSON struct returned from `go list`.
type PackageJSON struct {
Dir string
ImportPath string
ForTest string
// Source files
GoFiles [ ] string
CgoFiles [ ] string
CFiles [ ] string
// Dependency information
Imports [ ] string
ImportMap map [ string ] string
// Error information
Error * struct {
ImportStack [ ] string
Pos string
Err string
}
}
// Package holds a loaded package, its imports, and its parsed files.
type Package struct {
* Program
* packages . Package
Files [ ] * ast . File
Pkg * types . Package
types . Info
PackageJSON
program * Program
Files [ ] * ast . File
Pkg * types . Package
info types . Info
}
// Load loads the given package with all dependencies (including the runtime
// package). Call .Parse() afterwards to parse all Go files (including CGo
// processing, if necessary).
func ( p * Program ) Load ( importPath string ) error {
if p . Packages == nil {
p . Packages = make ( map [ string ] * Package )
func Load ( config * compileopts . Config , inputPkgs [ ] string , clangHeaders string , typeChecker types . Config ) ( * Program , error ) {
goroot , err := GetCachedGoroot ( config )
if err != nil {
return nil , err
}
err := p . loadPackage ( importPath )
wd , err := os . Getwd ( )
if err != nil {
return err
return nil , err
}
p . MainPkg = p . sorted [ len ( p . sorted ) - 1 ]
if _ , ok := p . Packages [ "runtime" ] ; ! ok {
// The runtime package wasn't loaded. Although `go list -deps` seems to
// return the full dependency list, there is no way to get those
// packages from the go/packages package. Therefore load the runtime
// manually and add it to the list of to-be-compiled packages
// (duplicates are already filtered).
return p . loadPackage ( "runtime" )
p := & Program {
config : config ,
clangHeaders : clangHeaders ,
typeChecker : typeChecker ,
goroot : goroot ,
workingDir : wd ,
Packages : make ( map [ string ] * Package ) ,
fset : token . NewFileSet ( ) ,
}
return nil
}
func ( p * Program ) loadPackage ( importPath string ) error {
cgoEnabled := "0"
if p . Build . CgoEnabled {
cgoEnabled = "1"
// List the dependencies of this package, in raw JSON format.
extraArgs := [ ] string { "-json" , "-deps" }
if config . TestConfig . CompileTestBinary {
extraArgs = append ( extraArgs , "-test" )
}
pkgs , err := packages . Load ( & packages . Config {
Mode : packages . NeedName | packages . NeedFiles | packages . NeedImports | packages . NeedDeps ,
Env : append ( os . Environ ( ) , "GOROOT=" + p . Build . GOROOT , "GOOS=" + p . Build . GOOS , "GOARCH=" + p . Build . GOARCH , "CGO_ENABLED=" + cgoEnabled ) ,
BuildFlags : [ ] string { "-tags" , strings . Join ( p . Build . BuildTags , " " ) } ,
Tests : p . Tests ,
} , importPath )
cmd , err := List ( config , extraArgs , inputPkgs )
if err != nil {
return err
return nil , err
}
var pkg * packages . Package
if p . Tests {
// We need the second package. Quoting from the docs:
// > For example, when using the go command, loading "fmt" with Tests=true
// > returns four packages, with IDs "fmt" (the standard package),
// > "fmt [fmt.test]" (the package as compiled for the test),
// > "fmt_test" (the test functions from source files in package fmt_test),
// > and "fmt.test" (the test binary).
pkg = pkgs [ 1 ]
} else {
if len ( pkgs ) != 1 {
return fmt . Errorf ( "expected exactly one package while importing %s, got %d" , importPath , len ( pkgs ) )
buf := & bytes . Buffer { }
cmd . Stdout = buf
cmd . Stderr = os . Stderr
err = cmd . Run ( )
if err != nil {
if exitErr , ok := err . ( * exec . ExitError ) ; ok {
if status , ok := exitErr . Sys ( ) . ( syscall . WaitStatus ) ; ok {
os . Exit ( status . ExitStatus ( ) )
}
os . Exit ( 1 )
}
pkg = pkgs [ 0 ]
}
var importError * Errors
var addPackages func ( pkg * packages . Package )
addPackages = func ( pkg * packages . Package ) {
if _ , ok := p . Packages [ pkg . PkgPath ] ; ok {
return
return nil , fmt . Errorf ( "failed to run `go list`: %s" , err )
}
// Parse the returned json from `go list`.
decoder := json . NewDecoder ( buf )
for {
pkg := & Package {
program : p ,
info : types . Info {
Types : make ( map [ ast . Expr ] types . TypeAndValue ) ,
Defs : make ( map [ * ast . Ident ] types . Object ) ,
Uses : make ( map [ * ast . Ident ] types . Object ) ,
Implicits : make ( map [ ast . Node ] types . Object ) ,
Scopes : make ( map [ ast . Node ] * types . Scope ) ,
Selections : make ( map [ * ast . SelectorExpr ] * types . Selection ) ,
} ,
}
pkg2 := p . newPackage ( pkg )
p . Packages [ pkg . PkgPath ] = pkg2
if len ( pkg . Errors ) != 0 {
if importError != nil {
// There was another error reported already. Do not report
// errors from multiple packages at once.
return
err := decoder . Decode ( & pkg . PackageJSON )
if err != nil {
if err == io . EOF {
break
}
importError = & Errors {
Pkg : pkg2 ,
return nil , err
}
if pkg . Error != nil {
// There was an error while importing (for example, a circular
// dependency).
pos := token . Position { }
fields := strings . Split ( pkg . Error . Pos , ":" )
if len ( fields ) >= 2 {
// There is some file/line/column information.
if n , err := strconv . Atoi ( fields [ len ( fields ) - 2 ] ) ; err == nil {
// Format: filename.go:line:colum
pos . Filename = strings . Join ( fields [ : len ( fields ) - 2 ] , ":" )
pos . Line = n
pos . Column , _ = strconv . Atoi ( fields [ len ( fields ) - 1 ] )
} else {
// Format: filename.go:line
pos . Filename = strings . Join ( fields [ : len ( fields ) - 1 ] , ":" )
pos . Line , _ = strconv . Atoi ( fields [ len ( fields ) - 1 ] )
}
pos . Filename = p . getOriginalPath ( pos . Filename )
}
for _ , err := range pkg . Errors {
pos := token . Position { }
fields := strings . Split ( err . Pos , ":" )
if len ( fields ) >= 2 {
// There is some file/line/column information.
if n , err := strconv . Atoi ( fields [ len ( fields ) - 2 ] ) ; err == nil {
// Format: filename.go:line:colum
pos . Filename = strings . Join ( fields [ : len ( fields ) - 2 ] , ":" )
pos . Line = n
pos . Column , _ = strconv . Atoi ( fields [ len ( fields ) - 1 ] )
} else {
// Format: filename.go:line
pos . Filename = strings . Join ( fields [ : len ( fields ) - 1 ] , ":" )
pos . Line , _ = strconv . Atoi ( fields [ len ( fields ) - 1 ] )
}
pos . Filename = p . getOriginalPath ( pos . Filename )
err := scanner . Error {
Pos : pos ,
Msg : pkg . Error . Err ,
}
if len ( pkg . Error . ImportStack ) != 0 {
return nil , Error {
ImportStack : pkg . Error . ImportStack ,
Err : err ,
}
importError . Errs = append ( importError . Errs , scanner . Error {
Pos : pos ,
Msg : err . Msg ,
} )
}
return
}
// Get the list of imports (sorted alphabetically).
names := make ( [ ] string , 0 , len ( pkg . Imports ) )
for name := range pkg . Imports {
names = append ( names , name )
}
sort . Strings ( names )
// Add all the imports.
for _ , name := range names {
addPackages ( pkg . Imports [ name ] )
return nil , err
}
p . sorted = append ( p . sorted , pkg2 )
}
addPackages ( pkg )
if importError != nil {
return * importError
p . sorted = append ( p . sorted , pkg )
p . Packages [ pkg . ImportPath ] = pkg
}
return nil
return p , nil
}
// getOriginalPath looks whether this path is in the generated GOROOT and if so,
@ -168,23 +181,23 @@ func (p *Program) loadPackage(importPath string) error {
// the input path is returned.
func ( p * Program ) getOriginalPath ( path string ) string {
originalPath := path
if strings . HasPrefix ( path , p . Build . GOROOT + string ( filepath . Separator ) ) {
if strings . HasPrefix ( path , p . goroot + string ( filepath . Separator ) ) {
// If this file is part of the synthetic GOROOT, try to infer the
// original path.
relpath := path [ len ( filepath . Join ( p . Build . GOROOT , "src" ) ) + 1 : ]
relpath := path [ len ( filepath . Join ( p . goroot , "src" ) ) + 1 : ]
realgorootPath := filepath . Join ( goenv . Get ( "GOROOT" ) , "src" , relpath )
if _ , err := os . Stat ( realgorootPath ) ; err == nil {
originalPath = realgorootPath
}
maybeInTinyGoRoot := false
for prefix := range pathsToOverride ( needsSyscallPackage ( p . Build . BuildTags ) ) {
for prefix := range pathsToOverride ( needsSyscallPackage ( p . config . BuildTags ( ) ) ) {
if ! strings . HasPrefix ( relpath , prefix ) {
continue
}
maybeInTinyGoRoot = true
}
if maybeInTinyGoRoot {
tinygoPath := filepath . Join ( p . TINYGOROOT , "src" , relpath )
tinygoPath := filepath . Join ( goenv . Get ( "TINYGOROOT" ) , "src" , relpath )
if _ , err := os . Stat ( tinygoPath ) ; err == nil {
originalPath = tinygoPath
}
@ -193,28 +206,18 @@ func (p *Program) getOriginalPath(path string) string {
return originalPath
}
// newPackage instantiates a new *Package object with initialized members.
func ( p * Program ) newPackage ( pkg * packages . Package ) * Package {
return & Package {
Program : p ,
Package : pkg ,
Info : types . Info {
Types : make ( map [ ast . Expr ] types . TypeAndValue ) ,
Defs : make ( map [ * ast . Ident ] types . Object ) ,
Uses : make ( map [ * ast . Ident ] types . Object ) ,
Implicits : make ( map [ ast . Node ] types . Object ) ,
Scopes : make ( map [ ast . Node ] * types . Scope ) ,
Selections : make ( map [ * ast . SelectorExpr ] * types . Selection ) ,
} ,
}
}
// Sorted returns a list of all packages, sorted in a way that no packages come
// before the packages they depend upon.
func ( p * Program ) Sorted ( ) [ ] * Package {
return p . sorted
}
// MainPkg returns the last package in the Sorted() slice. This is the main
// package of the program.
func ( p * Program ) MainPkg ( ) * Package {
return p . sorted [ len ( p . sorted ) - 1 ]
}
// Parse parses all packages and typechecks them.
//
// The returned error may be an Errors error, which contains a list of errors.
@ -222,22 +225,16 @@ func (p *Program) Sorted() []*Package {
// Idempotent.
func ( p * Program ) Parse ( ) error {
// Parse all packages.
for _ , pkg := range p . Sorted ( ) {
// TODO: do this in parallel.
for _ , pkg := range p . sorted {
err := pkg . Parse ( )
if err != nil {
return err
}
}
if p . Tests {
err := p . swapTestMain ( )
if err != nil {
return err
}
}
// Typecheck all packages.
for _ , pkg := range p . Sorted ( ) {
for _ , pkg := range p . sorted {
err := pkg . Check ( )
if err != nil {
return err
@ -247,82 +244,6 @@ func (p *Program) Parse() error {
return nil
}
func ( p * Program ) swapTestMain ( ) error {
var tests [ ] string
isTestFunc := func ( f * ast . FuncDecl ) bool {
// TODO: improve signature check
if strings . HasPrefix ( f . Name . Name , "Test" ) && f . Name . Name != "TestMain" {
return true
}
return false
}
for _ , f := range p . MainPkg . Files {
for i , d := range f . Decls {
switch v := d . ( type ) {
case * ast . FuncDecl :
if isTestFunc ( v ) {
tests = append ( tests , v . Name . Name )
}
if v . Name . Name == "main" {
// Remove main
if len ( f . Decls ) == 1 {
f . Decls = make ( [ ] ast . Decl , 0 )
} else {
f . Decls [ i ] = f . Decls [ len ( f . Decls ) - 1 ]
f . Decls = f . Decls [ : len ( f . Decls ) - 1 ]
}
}
}
}
}
// TODO: Check if they defined a TestMain and call it instead of testing.TestMain
const mainBody = ` package main
import (
"testing"
)
func main ( ) {
m := & testing . M {
Tests : [ ] testing . TestToCall {
{ { range . TestFunctions } }
{ Name : "{{.}}" , Func : { { . } } } ,
{ { end } }
} ,
}
testing . TestMain ( m )
}
`
tmpl := template . Must ( template . New ( "testmain" ) . Parse ( mainBody ) )
b := bytes . Buffer { }
tmplData := struct {
TestFunctions [ ] string
} {
TestFunctions : tests ,
}
err := tmpl . Execute ( & b , tmplData )
if err != nil {
return err
}
path := filepath . Join ( p . MainPkg . Dir , "$testmain.go" )
if p . fset == nil {
p . fset = token . NewFileSet ( )
}
newMain , err := parser . ParseFile ( p . fset , path , b . Bytes ( ) , parser . AllErrors )
if err != nil {
return err
}
p . MainPkg . Files = append ( p . MainPkg . Files , newMain )
return nil
}
// parseFile is a wrapper around parser.ParseFile.
func ( p * Program ) parseFile ( path string , mode parser . Mode ) ( * ast . File , error ) {
if p . fset == nil {
@ -342,14 +263,13 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
// Idempotent.
func ( p * Package ) Parse ( ) error {
if len ( p . Files ) != 0 {
return nil
return nil // nothing to do (?)
}
// Load the AST.
// TODO: do this in parallel.
if p . PkgPath == "unsafe" {
// Special case for the unsafe package. Don't even bother loading
// the files.
if p . ImportPath == "unsafe" {
// Special case for the unsafe package, which is defined internally by
// the types package.
p . Pkg = types . Unsafe
return nil
}
@ -369,11 +289,11 @@ func (p *Package) Parse() error {
// Idempotent.
func ( p * Package ) Check ( ) error {
if p . Pkg != nil {
return nil
return nil // already typechecked
}
var typeErrors [ ] error
checker := p . TypeChecker
checker := p . program . typeChecker // make a copy, because it will be modified
checker . Error = func ( err error ) {
typeErrors = append ( typeErrors , err )
}
@ -381,7 +301,7 @@ func (p *Package) Check() error {
// Do typechecking of the package.
checker . Importer = p
typesPkg , err := checker . Check ( p . Pkg Path, p . fset , p . Files , & p . I nfo)
typesPkg , err := checker . Check ( p . Import Path, p . program . fset , p . Files , & p . i nfo)
if err != nil {
if err , ok := err . ( Errors ) ; ok {
return err
@ -394,40 +314,46 @@ func (p *Package) Check() error {
// parseFiles parses the loaded list of files and returns this list.
func ( p * Package ) parseFiles ( ) ( [ ] * ast . File , error ) {
// TODO: do this concurrently.
var files [ ] * ast . File
var fileErrs [ ] error
var cgoFiles [ ] * ast . File
for _ , file := range p . GoFiles {
f , err := p . parseFile ( file , parser . ParseComments )
if err != nil {
fileErrs = append ( fileErrs , err )
continue
// Parse all files (incuding CgoFiles).
parseFile := func ( file string ) {
if ! filepath . IsAbs ( file ) {
file = filepath . Join ( p . Dir , file )
}
f , err := p . program . parseFile ( file , parser . ParseComments )
if err != nil {
fileErrs = append ( fileErrs , err )
continue
}
for _ , importSpec := range f . Imports {
if importSpec . Path . Value == ` "C" ` {
cgoFiles = append ( cgoFiles , f )
}
return
}
files = append ( files , f )
}
if len ( cgoFiles ) != 0 {
cflags := append ( p . CFlags , "-I" + filepath . Dir ( p . GoFiles [ 0 ] ) )
if p . ClangHeaders != "" {
cflags = append ( cflags , "-Xclang" , "-internal-isystem" , "-Xclang" , p . ClangHeaders )
for _ , file := range p . GoFiles {
parseFile ( file )
}
for _ , file := range p . CgoFiles {
parseFile ( file )
}
// Do CGo processing.
if len ( p . CgoFiles ) != 0 {
var cflags [ ] string
cflags = append ( cflags , p . program . config . CFlags ( ) ... )
cflags = append ( cflags , "-I" + p . Dir )
if p . program . clangHeaders != "" {
cflags = append ( cflags , "-Xclang" , "-internal-isystem" , "-Xclang" , p . program . clangHeaders )
}
generated , ldflags , errs := cgo . Process ( files , p . Program . Dir , p . fset , cflags )
generated , ldflags , errs := cgo . Process ( files , p . p rogram. working Dir, p . program . fset , cflags )
if errs != nil {
fileErrs = append ( fileErrs , errs ... )
}
files = append ( files , generated )
p . LDFlags = append ( p . LDFlags , ldflags ... )
p . program . LDFlags = append ( p . program . LDFlags , ldflags ... )
}
// Only return an error after CGo processing, so that errors in parsing and
// CGo can be reported together.
if len ( fileErrs ) != 0 {
return nil , Errors { p , fileErrs }
}
@ -441,8 +367,13 @@ func (p *Package) Import(to string) (*types.Package, error) {
if to == "unsafe" {
return types . Unsafe , nil
}
if _ , ok := p . Imports [ to ] ; ok {
return p . Packages [ p . Imports [ to ] . PkgPath ] . Pkg , nil
if replace , ok := p . ImportMap [ to ] ; ok {
// This import path should be replaced by another import path, according
// to `go list`.
to = replace
}
if imported , ok := p . program . Packages [ to ] ; ok {
return imported . Pkg , nil
} else {
return nil , errors . New ( "package not imported: " + to )
}
Ayke van Laethem
4 years ago
Ron Evans