From e10d05c74fca049207a89939581cea1087457423 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Sun, 25 Nov 2018 17:37:31 +0100 Subject: [PATCH] loader: switch to custom program loader --- Makefile | 2 +- compiler/compiler.go | 43 ++++-- ir/ir.go | 7 +- loader/errors.go | 27 ++++ loader/loader.go | 333 +++++++++++++++++++++++++++++++++++++++++++ loader/ssa.go | 18 +++ main.go | 6 + 7 files changed, 416 insertions(+), 20 deletions(-) create mode 100644 loader/errors.go create mode 100644 loader/loader.go create mode 100644 loader/ssa.go diff --git a/Makefile b/Makefile index 2eacb156..dabafd1f 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ clean: @rm -rf build fmt: - @go fmt . ./compiler ./interp ./ir ./src/device/arm ./src/examples/* ./src/machine ./src/runtime ./src/sync + @go fmt . ./compiler ./interp ./loader ./ir ./src/device/arm ./src/examples/* ./src/machine ./src/runtime ./src/sync @go fmt ./testdata/*.go test: diff --git a/compiler/compiler.go b/compiler/compiler.go index c3cf1ddf..e8223727 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -5,7 +5,6 @@ import ( "fmt" "go/build" "go/constant" - "go/parser" "go/token" "go/types" "os" @@ -17,7 +16,7 @@ import ( "github.com/aykevl/go-llvm" "github.com/aykevl/tinygo/ir" - "golang.org/x/tools/go/loader" + "github.com/aykevl/tinygo/loader" "golang.org/x/tools/go/ssa" ) @@ -183,14 +182,11 @@ func (c *Compiler) Compile(mainPath string) error { gopath = runtime.GOROOT() + string(filepath.ListSeparator) + gopath } - config := loader.Config{ - TypeChecker: types.Config{ - Sizes: &StdSizes{ - IntSize: int64(c.targetData.TypeAllocSize(c.intType)), - PtrSize: int64(c.targetData.PointerSize()), - MaxAlign: int64(c.targetData.PrefTypeAlignment(c.i8ptrType)), - }, - }, + wd, err := os.Getwd() + if err != nil { + return err + } + lprogram := &loader.Program{ Build: &build.Context{ GOARCH: tripleSplit[0], GOOS: tripleSplit[2], @@ -201,15 +197,32 @@ func (c *Compiler) Compile(mainPath string) error { Compiler: "gc", // must be one of the recognized compilers BuildTags: append([]string{"tinygo", "gc." + c.selectGC()}, c.BuildTags...), }, - ParserMode: parser.ParseComments, + TypeChecker: types.Config{ + Sizes: &StdSizes{ + IntSize: int64(c.targetData.TypeAllocSize(c.intType)), + PtrSize: int64(c.targetData.PointerSize()), + MaxAlign: int64(c.targetData.PrefTypeAlignment(c.i8ptrType)), + }, + }, + Dir: wd, } - config.Import("runtime") if strings.HasSuffix(mainPath, ".go") { - config.CreateFromFilenames("main", mainPath) + _, err = lprogram.ImportFile(mainPath) + if err != nil { + return err + } } else { - config.Import(mainPath) + _, err = lprogram.Import(mainPath, wd) + if err != nil { + return err + } + } + _, err = lprogram.Import("runtime", "") + if err != nil { + return err } - lprogram, err := config.Load() + + err = lprogram.Parse() if err != nil { return err } diff --git a/ir/ir.go b/ir/ir.go index d0028610..65558b8a 100644 --- a/ir/ir.go +++ b/ir/ir.go @@ -8,9 +8,8 @@ import ( "strings" "github.com/aykevl/go-llvm" - "golang.org/x/tools/go/loader" + "github.com/aykevl/tinygo/loader" "golang.org/x/tools/go/ssa" - "golang.org/x/tools/go/ssa/ssautil" ) // This file provides a wrapper around go/ssa values and adds extra @@ -78,7 +77,7 @@ type Interface struct { // Create and intialize a new *Program from a *ssa.Program. func NewProgram(lprogram *loader.Program, mainPath string) *Program { comments := map[string]*ast.CommentGroup{} - for _, pkgInfo := range lprogram.AllPackages { + for _, pkgInfo := range lprogram.Sorted() { for _, file := range pkgInfo.Files { for _, decl := range file.Decls { switch decl := decl.(type) { @@ -106,7 +105,7 @@ func NewProgram(lprogram *loader.Program, mainPath string) *Program { } } - program := ssautil.CreateProgram(lprogram, ssa.SanityCheckFunctions|ssa.BareInits|ssa.GlobalDebug) + program := lprogram.LoadSSA() program.Build() // Find the main package, which is a bit difficult when running a .go file diff --git a/loader/errors.go b/loader/errors.go new file mode 100644 index 00000000..b8e4860a --- /dev/null +++ b/loader/errors.go @@ -0,0 +1,27 @@ +package loader + +// Errors contains a list of parser errors or a list of typechecker errors for +// the given package. +type Errors struct { + Pkg *Package + Errs []error +} + +func (e Errors) Error() string { + return "could not compile: " + e.Errs[0].Error() +} + +// ImportCycleErrors is returned when encountering an import cycle. The list of +// packages is a list from the root package to the leaf package that imports one +// of the packages in the list. +type ImportCycleError struct { + Packages []string +} + +func (e *ImportCycleError) Error() string { + msg := "import cycle: " + e.Packages[0] + for _, path := range e.Packages[1:] { + msg += " → " + path + } + return msg +} diff --git a/loader/loader.go b/loader/loader.go new file mode 100644 index 00000000..26a3fb03 --- /dev/null +++ b/loader/loader.go @@ -0,0 +1,333 @@ +package loader + +import ( + "errors" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "sort" +) + +// Program holds all packages and some metadata about the program as a whole. +type Program struct { + Build *build.Context + Packages map[string]*Package + sorted []*Package + fset *token.FileSet + TypeChecker types.Config + Dir string // current working directory (for error reporting) +} + +// Package holds a loaded package, its imports, and its parsed files. +type Package struct { + *Program + *build.Package + Imports map[string]*Package + Importing bool + Files []*ast.File + Pkg *types.Package + types.Info +} + +// Import loads the given package relative to srcDir (for the vendor directory). +// It only loads the current package without recursion. +func (p *Program) Import(path, srcDir string) (*Package, error) { + if p.Packages == nil { + p.Packages = make(map[string]*Package) + } + + // Load this package. + buildPkg, err := p.Build.Import(path, srcDir, build.ImportComment) + if err != nil { + return nil, err + } + if existingPkg, ok := p.Packages[buildPkg.ImportPath]; ok { + // Already imported, or at least started the import. + return existingPkg, nil + } + p.sorted = nil // invalidate the sorted order of packages + pkg := p.newPackage(buildPkg) + p.Packages[buildPkg.ImportPath] = pkg + return pkg, nil +} + +// ImportFile loads and parses the import statements in the given path and +// creates a pseudo-package out of it. +func (p *Program) ImportFile(path string) (*Package, error) { + if p.Packages == nil { + p.Packages = make(map[string]*Package) + } + if _, ok := p.Packages[path]; ok { + // unlikely + return nil, errors.New("loader: cannot import file that is already imported as package: " + path) + } + + file, err := p.parseFile(path, parser.ImportsOnly) + if err != nil { + return nil, err + } + buildPkg := &build.Package{ + Dir: filepath.Dir(path), + ImportPath: path, + GoFiles: []string{filepath.Base(path)}, + } + for _, importSpec := range file.Imports { + buildPkg.Imports = append(buildPkg.Imports, importSpec.Path.Value[1:len(importSpec.Path.Value)-1]) + } + p.sorted = nil // invalidate the sorted order of packages + pkg := p.newPackage(buildPkg) + p.Packages[buildPkg.ImportPath] = pkg + return pkg, nil +} + +// newPackage instantiates a new *Package object with initialized members. +func (p *Program) newPackage(pkg *build.Package) *Package { + return &Package{ + Program: p, + Package: pkg, + Imports: make(map[string]*Package, len(pkg.Imports)), + Info: types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Scopes: make(map[ast.Node]*types.Scope), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + }, + } +} + +// Sorted returns a list of all packages, sorted in a way that no packages come +// before the packages they depend upon. +func (p *Program) Sorted() []*Package { + if p.sorted == nil { + p.sort() + } + return p.sorted +} + +func (p *Program) sort() { + p.sorted = nil + packageList := make([]*Package, 0, len(p.Packages)) + packageSet := make(map[string]struct{}, len(p.Packages)) + worklist := make([]string, 0, len(p.Packages)) + for path := range p.Packages { + worklist = append(worklist, path) + } + sort.Strings(worklist) + for len(worklist) != 0 { + pkgPath := worklist[0] + pkg := p.Packages[pkgPath] + + if _, ok := packageSet[pkgPath]; ok { + // Package already in the final package list. + worklist = worklist[1:] + continue + } + + unsatisfiedImports := make([]string, 0) + for _, pkg := range pkg.Imports { + if _, ok := packageSet[pkg.ImportPath]; ok { + continue + } + unsatisfiedImports = append(unsatisfiedImports, pkg.ImportPath) + } + sort.Strings(unsatisfiedImports) + if len(unsatisfiedImports) == 0 { + // All dependencies of this package are satisfied, so add this + // package to the list. + packageList = append(packageList, pkg) + packageSet[pkgPath] = struct{}{} + worklist = worklist[1:] + } else { + // Prepend all dependencies to the worklist and reconsider this + // package (by not removing it from the worklist). At that point, it + // must be possible to add it to packageList. + worklist = append(unsatisfiedImports, worklist...) + } + } + + p.sorted = packageList +} + +// Parse recursively imports all packages, parses them, and typechecks them. +// +// The returned error may be an Errors error, which contains a list of errors. +// +// Idempotent. +func (p *Program) Parse() error { + // Load all imports + for _, pkg := range p.Sorted() { + err := pkg.importRecursively() + if err != nil { + if err, ok := err.(*ImportCycleError); ok { + err.Packages = append([]string{pkg.ImportPath}, err.Packages...) + } + return err + } + } + + // Parse all packages. + for _, pkg := range p.Sorted() { + err := pkg.Parse() + if err != nil { + return err + } + } + + // Typecheck all packages. + for _, pkg := range p.Sorted() { + err := pkg.Check() + if err != nil { + return err + } + } + + return nil +} + +// parseFile is a wrapper around parser.ParseFile. +func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) { + if p.fset == nil { + p.fset = token.NewFileSet() + } + + rd, err := os.Open(path) + if err != nil { + return nil, err + } + defer rd.Close() + relpath := path + if filepath.IsAbs(path) { + relpath, err = filepath.Rel(p.Dir, path) + if err != nil { + return nil, err + } + } + return parser.ParseFile(p.fset, relpath, rd, mode) +} + +// Parse parses and typechecks this package. +// +// Idempotent. +func (p *Package) Parse() error { + if len(p.Files) != 0 { + return nil + } + + // Load the AST. + // TODO: do this in parallel. + if p.ImportPath == "unsafe" { + // Special case for the unsafe package. Don't even bother loading + // the files. + p.Pkg = types.Unsafe + return nil + } + + files, err := p.parseFiles() + if err != nil { + return err + } + p.Files = files + + return nil +} + +// Check runs the package through the typechecker. The package must already be +// loaded and all dependencies must have been checked already. +// +// Idempotent. +func (p *Package) Check() error { + if p.Pkg != nil { + return nil + } + + var typeErrors []error + checker := p.TypeChecker + checker.Error = func(err error) { + typeErrors = append(typeErrors, err) + } + + // Do typechecking of the package. + checker.Importer = p + + typesPkg, err := checker.Check(p.ImportPath, p.fset, p.Files, &p.Info) + if err != nil { + if err, ok := err.(Errors); ok { + return err + } + return Errors{p, typeErrors} + } + p.Pkg = typesPkg + return nil +} + +// parseFiles parses the loaded list of files and returns this list. +func (p *Package) parseFiles() ([]*ast.File, error) { + if len(p.CgoFiles) != 0 { + return nil, errors.New("loader: todo cgo: " + p.CgoFiles[0]) + } + + // TODO: do this concurrently. + var files []*ast.File + var fileErrs []error + for _, file := range p.GoFiles { + f, err := p.parseFile(filepath.Join(p.Package.Dir, file), parser.ParseComments) + if err != nil { + fileErrs = append(fileErrs, err) + } else { + files = append(files, f) + } + } + if len(fileErrs) != 0 { + return nil, Errors{p, fileErrs} + } + return files, nil +} + +// Import implements types.Importer. It loads and parses packages it encounters +// along the way, if needed. +func (p *Package) Import(to string) (*types.Package, error) { + if to == "unsafe" { + return types.Unsafe, nil + } + if _, ok := p.Imports[to]; ok { + return p.Imports[to].Pkg, nil + } else { + panic("package not imported: " + to) + } +} + +// importRecursively calls Program.Import() on all imported packages, and calls +// importRecursively() on the imported packages as well. +// +// Idempotent. +func (p *Package) importRecursively() error { + p.Importing = true + for _, to := range p.Package.Imports { + if _, ok := p.Imports[to]; ok { + continue + } + importedPkg, err := p.Program.Import(to, p.Package.Dir) + if err != nil { + if err, ok := err.(*ImportCycleError); ok { + err.Packages = append([]string{p.ImportPath}, err.Packages...) + } + return err + } + if importedPkg.Importing { + return &ImportCycleError{[]string{p.ImportPath, importedPkg.ImportPath}} + } + err = importedPkg.importRecursively() + if err != nil { + return err + } + p.Imports[to] = importedPkg + } + p.Importing = false + return nil +} diff --git a/loader/ssa.go b/loader/ssa.go new file mode 100644 index 00000000..a047d35d --- /dev/null +++ b/loader/ssa.go @@ -0,0 +1,18 @@ +package loader + +import ( + "golang.org/x/tools/go/ssa" +) + +// LoadSSA constructs the SSA form of the loaded packages. +// +// The program must already be parsed and type-checked with the .Parse() method. +func (p *Program) LoadSSA() *ssa.Program { + prog := ssa.NewProgram(p.fset, ssa.SanityCheckFunctions|ssa.BareInits|ssa.GlobalDebug) + + for _, pkg := range p.Sorted() { + prog.CreatePackage(pkg.Pkg, pkg.Files, &pkg.Info, true) + } + + return prog +} diff --git a/main.go b/main.go index 8e8fc43e..77c76fb3 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/aykevl/go-llvm" "github.com/aykevl/tinygo/compiler" "github.com/aykevl/tinygo/interp" + "github.com/aykevl/tinygo/loader" ) var commands = map[string]string{ @@ -454,6 +455,11 @@ func handleCompilerError(err error) { fmt.Fprintln(os.Stderr) } else if errCompiler, ok := err.(types.Error); ok { fmt.Fprintln(os.Stderr, errCompiler) + } else if errLoader, ok := err.(loader.Errors); ok { + fmt.Fprintln(os.Stderr, "#", errLoader.Pkg.ImportPath) + for _, err := range errLoader.Errs { + fmt.Fprintln(os.Stderr, err) + } } else { fmt.Fprintln(os.Stderr, "error:", err) }