cue / pkg / gen.go

// Copyright 2018 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build ignore

// gen.go generates the pkg.go files inside the packages under the pkg directory.
//
// It takes the list of packages from the packages.txt.
//
// Be sure to also update an entry in pkg/pkg.go, if so desired.
package main

import (
	"bytes"
	"cmp"
	_ "embed"
	"flag"
	"fmt"
	"go/constant"
	"go/format"
	"go/token"
	"go/types"
	"log"
	"math/big"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"text/template"

	"golang.org/x/tools/go/packages"

	"cuelang.org/go/cue/ast"
	cueformat "cuelang.org/go/cue/format"
	"cuelang.org/go/cue/parser"
	"cuelang.org/go/internal"
)

type headerParams struct {
	GoPkg  string
	CUEPkg string

	PackageDoc  string
	PackageDefs string
}

var header = template.Must(template.New("").Parse(
	`// Code generated by cuelang.org/go/pkg/gen. DO NOT EDIT.

{{if .PackageDoc}}
{{.PackageDoc -}}
//     {{.PackageDefs}}
{{end -}}
package {{.GoPkg}}

{{if .CUEPkg -}}
import (
	"cuelang.org/go/internal/core/adt"
	"cuelang.org/go/internal/pkg"
)

func init() {
	pkg.Register({{printf "%q" .CUEPkg}}, p)
}

var _ = adt.TopKind // in case the adt package isn't used
{{end}}
`))

const pkgParent = "cuelang.org/go/pkg"

func main() {
	flag.Parse()
	log.SetFlags(log.Lshortfile)
	log.SetOutput(os.Stdout)

	cfg := &packages.Config{Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes}
	pkgs, err := packages.Load(cfg, "./...")
	if err != nil {
		fmt.Fprintf(os.Stderr, "load: %v\n", err)
		os.Exit(1)
	}
	if packages.PrintErrors(pkgs) > 0 {
		os.Exit(1)
	}
	// Sort the Go packages by import path; otherwise adding a new builtin package
	// puts it at the very end of the list the first time it is getting added to register.go,
	// as it's not imported by the root package yet. Sorting ensures consistent output.
	slices.SortFunc(pkgs, func(a, b *packages.Package) int {
		return cmp.Compare(a.PkgPath, b.PkgPath)
	})

	regBuf := new(bytes.Buffer)
	fmt.Fprintf(regBuf, "// Code generated by cuelang.org/go/pkg/gen. DO NOT EDIT.\n\n")
	fmt.Fprintf(regBuf, "package pkg\n\n")
	fmt.Fprintf(regBuf, "import (\n")
	for _, pkg := range pkgs {
		switch {
		case pkg.PkgPath == pkgParent:
			// The pkg package itself should not be generated.
		case strings.Contains(pkg.PkgPath, "/internal"):
			// Internal packages are not for public use.
		default:
			fmt.Fprintf(regBuf, "\t_ %q\n", pkg.PkgPath)
			if pkg.PkgPath == "cuelang.org/go/pkg/path" {
				// TODO remove this special case. Currently the path
				// pkg.go file cannot be generated automatically but that
				// will be possible when we can attach arbitrary signatures
				// to builtin functions.
				break
			}
			if err := generate(pkg); err != nil {
				log.Fatalf("%s: %v", pkg, err)
			}
		}
	}
	fmt.Fprintf(regBuf, ")\n")
	if err := os.WriteFile("register.go", regBuf.Bytes(), 0o666); err != nil {
		log.Fatal(err)
	}
}

type generator struct {
	dir         string
	w           *bytes.Buffer
	cuePkgPath  string
	first       bool
	nonConcrete bool
}

func generate(pkg *packages.Package) error {
	// go/packages supports multiple build systems, including some which don't keep
	// a Go package entirely within a single directory.
	// However, we know for certain that CUE uses modules, so it is the case here.
	// We can figure out the directory from the first Go file.
	pkgDir := filepath.Dir(pkg.GoFiles[0])
	cuePkg := strings.TrimPrefix(pkg.PkgPath, pkgParent+"/")
	g := generator{
		dir:        pkgDir,
		cuePkgPath: cuePkg,
		w:          &bytes.Buffer{},
	}

	params := headerParams{
		GoPkg:  pkg.Name,
		CUEPkg: cuePkg,
	}
	// As a special case, the "tool" package cannot be imported from CUE.
	skipRegister := params.CUEPkg == "tool"
	if skipRegister {
		params.CUEPkg = ""
	}

	if doc, err := os.ReadFile(filepath.Join(pkgDir, "doc.txt")); err == nil {
		defs, err := os.ReadFile(filepath.Join(pkgDir, pkg.Name+".cue"))
		if err != nil {
			return err
		}
		i := bytes.Index(defs, []byte("package "+pkg.Name))
		defs = defs[i+len("package "+pkg.Name)+1:]
		defs = bytes.TrimRight(defs, "\n")
		defs = bytes.ReplaceAll(defs, []byte("\n"), []byte("\n//\t"))
		params.PackageDoc = string(doc)
		params.PackageDefs = string(defs)
	}

	if err := header.Execute(g.w, params); err != nil {
		return err
	}

	if !skipRegister {
		fmt.Fprintf(g.w, "var p = &pkg.Package{\nNative: []*pkg.Builtin{")
		g.first = true
		if err := g.processGo(pkg); err != nil {
			return err
		}
		fmt.Fprintf(g.w, "},\n")
		if err := g.processCUE(); err != nil {
			return err
		}
		fmt.Fprintf(g.w, "}\n")
	}

	b, err := format.Source(g.w.Bytes())
	if err != nil {
		fmt.Printf("go/format error on %s: %v\n", pkg.PkgPath, err)
		b = g.w.Bytes() // write the unformatted source
	}

	filename := filepath.Join(pkgDir, "pkg.go")

	if err := os.WriteFile(filename, b, 0666); err != nil {
		return err
	}
	return nil
}

func (g *generator) sep() {
	if g.first {
		g.first = false
		return
	}
	fmt.Fprint(g.w, ", ")
}

// processCUE mixes in CUE definitions defined in the package directory.
func (g *generator) processCUE() error {
	// Note: we avoid using the cue/load and the cuecontext packages
	// because they depend on the standard library which is what this
	// command is generating - cyclic dependencies are undesirable in general.
	// We only need to load the declarations from one CUE file if it exists.
	expr, err := loadCUEDecls(g.dir)
	if err != nil {
		return fmt.Errorf("error processing %s: %v", g.cuePkgPath, err)
	}
	if expr == nil { // No syntax to add.
		return nil
	}
	b, err := cueformat.Node(expr)
	if err != nil {
		return err
	}

	// Compact the CUE by removing empty lines. This requires re-formatting to align fields.
	// TODO(mvdan): provide a "compact" option in cue/format for this purpose?
	b = bytes.ReplaceAll(b, []byte("\n\n"), []byte("\n"))
	b, err = cueformat.Source(b)
	if err != nil {
		return err
	}
	b = bytes.TrimSpace(b) // no trailing newline

	// Try to use a Go string with backquotes, for readability.
	// If not possible due to cueSrc itself having backquotes,
	// use a single-line double-quoted string, removing tabs for brevity.
	// We don't use strconv.CanBackquote as it is for quoting as a single line.
	if cueSrc := string(b); !strings.Contains(cueSrc, "`") {
		fmt.Fprintf(g.w, "CUE: `%s`,\n", cueSrc)
	} else {
		cueSrc = strings.ReplaceAll(cueSrc, "\t", "")
		fmt.Fprintf(g.w, "CUE: %q,\n", cueSrc)
	}
	return nil
}

func (g *generator) processGo(pkg *packages.Package) error {
	// We sort the objects by their original source code position.
	// Otherwise, go/types defaults to sorting by name strings.
	// We could remove this code if we were fine with sorting by name.
	scope := pkg.Types.Scope()
	type objWithPos struct {
		obj types.Object
		pos token.Position
	}
	var objs []objWithPos
	for _, name := range scope.Names() {
		obj := scope.Lookup(name)
		objs = append(objs, objWithPos{obj, pkg.Fset.Position(obj.Pos())})
	}
	slices.SortFunc(objs, func(a, b objWithPos) int {
		if c := cmp.Compare(a.pos.Filename, b.pos.Filename); c != 0 {
			return c
		}
		return cmp.Compare(a.pos.Line, b.pos.Line)
	})

	for _, obj := range objs {
		obj := obj.obj // no longer need the token.Position
		if !obj.Exported() {
			continue
		}
		// TODO: support type declarations.
		switch obj := obj.(type) {
		case *types.Const:
			var value string
			switch v := obj.Val(); v.Kind() {
			case constant.Bool, constant.Int, constant.String:
				// TODO: convert octal numbers
				value = v.ExactString()
			case constant.Float:
				var rat big.Rat
				rat.SetString(v.ExactString())
				var float big.Float
				float.SetRat(&rat)
				value = float.Text('g', -1)
			default:
				fmt.Printf("Dropped entry %s.%s (%T: %v)\n", g.cuePkgPath, obj.Name(), v.Kind(), v.ExactString())
				continue
			}
			g.sep()
			fmt.Fprintf(g.w, "{\nName: %q,\n Const: %q,\n}", obj.Name(), value)
		case *types.Func:
			g.genFunc(obj)
		}
	}
	return nil
}

var (
	typeError = types.Universe.Lookup("error").Type()
	typeByte  = types.Universe.Lookup("byte").Type()
)

func (g *generator) genFunc(fn *types.Func) {
	g.nonConcrete = false
	sign := fn.Signature()
	if sign.Recv() != nil {
		return
	}
	params := sign.Params()
	results := sign.Results()
	if results == nil || (results.Len() != 1 && results.At(1).Type() != typeError) {
		fmt.Printf("Dropped func %s.%s: must have one return value or a value and an error %v\n", g.cuePkgPath, fn.Name(), sign)
		return
	}

	g.sep()
	fmt.Fprintf(g.w, "{\n")
	defer fmt.Fprintf(g.w, "}")

	fmt.Fprintf(g.w, "Name: %q,\n", fn.Name())

	needCallContext := false
	args := []string{}
	vals := []string{}
	kind := []string{}
	for param := range params.Variables() {
		typ := param.Type()
		if typ.String() == "cuelang.org/go/internal/pkg.Schema" {
			needCallContext = true
		}
		methodName := g.callCtxtGetter(typ)
		argKind := g.adtKind(param.Type())
		vals = append(vals, fmt.Sprintf("c.%s(%d)", methodName, len(args)))
		args = append(args, param.Name())
		kind = append(kind, argKind)
	}

	fmt.Fprintf(g.w, "Params: []pkg.Param{\n")
	for _, k := range kind {
		fmt.Fprintf(g.w, "{Kind: %s},\n", k)
	}
	fmt.Fprintf(g.w, "\n},\n")

	fmt.Fprintf(g.w, "Result: %s,\n", g.adtKind(results.At(0).Type()))
	if g.nonConcrete {
		fmt.Fprintf(g.w, "NonConcrete: true,\n")
	}

	argList := strings.Join(args, ", ")
	valList := strings.Join(vals, ", ")
	init := ""
	if len(args) > 0 {
		init = fmt.Sprintf("%s := %s", argList, valList)
	}

	name := fn.Name()
	if needCallContext {
		argList = "c.OpContext(), " + argList

		// Main function is used for Godoc documentation. Once we have proper
		// CUE function signatures, we can remove these stubs.
		// NOTE: this will not work for scripts that are not cased. But this
		// is intended to be a temporary situation anyway.
		name = strings.ToLower(name[:1]) + name[1:]
	}

	fmt.Fprintf(g.w, "Func: func(c *pkg.CallCtxt) {")
	defer fmt.Fprintln(g.w, "},")
	fmt.Fprintln(g.w)
	if init != "" {
		fmt.Fprintln(g.w, init)
	}
	fmt.Fprintln(g.w, "if c.Do() {")
	defer fmt.Fprintln(g.w, "}")
	if results.Len() == 1 {
		fmt.Fprintf(g.w, "c.Ret = %s(%s)", name, argList)
	} else {
		fmt.Fprintf(g.w, "c.Ret, c.Err = %s(%s)", name, argList)
	}
}

// callCtxtGetter returns the name of the [cuelang.org/go/internal/pkg.CallCtxt] method
// which can be used to fetch a parameter of the given type.
func (g *generator) callCtxtGetter(typ types.Type) string {
	switch typ := typ.(type) {
	case *types.Basic:
		return strings.Title(typ.String()) // "int" turns into "Int"
	case *types.Slice:
		switch typ.Elem().String() {
		case "byte":
			return "Bytes"
		case "string":
			return "StringList"
		case "*cuelang.org/go/internal.Decimal":
			return "DecimalList"
		}
		return "List"
	}
	switch typ.String() {
	case "*math/big.Int":
		return "BigInt"
	case "*math/big.Float":
		return "BigFloat"
	case "*cuelang.org/go/internal.Decimal":
		return "Decimal"
	case "cuelang.org/go/internal/pkg.List":
		return "CueList"
	case "cuelang.org/go/internal/pkg.Struct":
		return "Struct"
	case "cuelang.org/go/cue.Value",
		"cuelang.org/go/cue/ast.Expr":
		return "Value"
	case "cuelang.org/go/internal/pkg.Schema":
		g.nonConcrete = true
		return "Schema"
	case "io.Reader":
		return "Reader"
	}
	log.Fatal("callCtxtGetter: unhandled Go type ", typ.String())
	return ""
}

// adtKind provides a Go expression string which describes
// a [cuelang.org/go/internal/core/adt.Kind] value for the given type.
func (g *generator) adtKind(typ types.Type) string {
	// TODO: detect list and structs types for return values.
	switch typ := typ.(type) {
	case *types.Slice:
		if typ.Elem() == typeByte {
			return "adt.BytesKind | adt.StringKind"
		}
		return "adt.ListKind"
	case *types.Map:
		return "adt.StructKind"
	case *types.Basic:
		if typ.Info()&types.IsInteger != 0 {
			return "adt.IntKind"
		}
		if typ.Kind() == types.Float64 {
			return "adt.NumberKind"
		}
		return "adt." + strings.Title(typ.String()) + "Kind" // "bool" turns into "adt.BoolKind"
	}
	switch typ.String() {
	case "error":
		return "adt.BottomKind"
	case "io.Reader":
		return "adt.BytesKind | adt.StringKind"
	case "cuelang.org/go/internal/pkg.Struct":
		return "adt.StructKind"
	case "cuelang.org/go/internal/pkg.List":
		return "adt.ListKind"
	case "*math/big.Int":
		return "adt.IntKind"
	case "*cuelang.org/go/internal.Decimal", "*math/big.Float":
		return "adt.NumberKind"
	case "cuelang.org/go/cue.Value", "cuelang.org/go/cue/ast.Expr", "cuelang.org/go/internal/pkg.Schema":
		return "adt.TopKind" // TODO: can be more precise

	// Some builtin functions return custom types, like [cuelang.org/go/pkg/time.Split].
	// TODO: we can simplify this once the CUE API declarations in ./pkg/...
	// use CUE function signatures to validate their parameters and results.
	case "*cuelang.org/go/pkg/time.Parts":
		return "adt.StructKind"
	case "*cuelang.org/go/pkg/net.ParsedCIDR":
		return "adt.StructKind"
	}
	log.Fatal("adtKind: unhandled Go type ", typ.String())
	return ""
}

// loadCUEDecls parses a single CUE file from a directory and returns its contents
// as an expression, typically a struct holding all of a file's declarations.
// If there are no CUE files, it returns (nil, nil).
func loadCUEDecls(dir string) (ast.Expr, error) {
	cuefiles, err := filepath.Glob(filepath.Join(dir, "*.cue"))
	if err != nil || len(cuefiles) == 0 {
		return nil, err
	}
	if len(cuefiles) == 0 {
		return nil, nil
	}
	if len(cuefiles) > 1 {
		// Supporting multiple CUE files would require merging declarations.
		return nil, fmt.Errorf("multiple CUE files not supported in this generator")
	}
	src, err := os.ReadFile(cuefiles[0])
	if err != nil {
		return nil, err
	}
	file, err := parser.ParseFile(cuefiles[0], src)
	if err != nil {
		return nil, err
	}
	return internal.ToExpr(file), nil
}