Something went wrong. Try again.
A native-speed rewrite of the roblox-ts compiler in golang
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831package encoder
import ( "cmp" "encoding/binary" "fmt" "slices" "sync"
"rotor/tsgo/ast" "rotor/tsgo/core" "github.com/zeebo/xxh3")
func init() { if ast.KindLastUnaryOperator > 0x3f { panic(fmt.Sprintf("KindLastUnaryOperator (%d) exceeds the 6-bit commonData capacity (max 63)", ast.KindLastUnaryOperator)) }}
const ( NodeOffsetKind = iota * 4 NodeOffsetPos NodeOffsetEnd NodeOffsetNext NodeOffsetParent NodeOffsetData NodeOffsetFlags // NodeSize is the number of bytes that represents a single node in the encoded format. NodeSize)
const ( NodeDataTypeChildren uint32 = iota << 30 NodeDataTypeString NodeDataTypeExtendedData)
const ( NodeDataTypeMask uint32 = 0xc0_00_00_00 NodeDataChildMask uint32 = 0x00_00_00_ff NodeDataStringIndexMask uint32 = 0x00_ff_ff_ff)
const ( SyntaxKindNodeList uint32 = 1<<32 - 1)
const ( HeaderOffsetMetadata = iota * 4 HeaderOffsetHashLo0 HeaderOffsetHashLo1 HeaderOffsetHashHi0 HeaderOffsetHashHi1 HeaderOffsetParseOptions HeaderOffsetStringOffsets HeaderOffsetStringData HeaderOffsetExtendedData HeaderOffsetStructuredData HeaderOffsetNodes HeaderSize)
const ( ProtocolVersion uint8 = 5)
// Source File Binary Format// =========================//// The following defines a protocol for serializing TypeScript SourceFile objects to a compact binary format. All integer// values are little-endian.//// Overview// --------//// The format comprises seven sections://// | Section | Length | Description |// | ------------------ | ------------------ | ----------------------------------------------------------------------------------------------- |// | Header | 44 bytes | Contains the content hash, parse options, flags, and byte offsets to the start of each section. |// | String offsets | 8 bytes per string | Pairs of starting byte offsets and ending byte offsets into the **string data** section. |// | String data | variable | UTF-8 encoded string data. |// | Extended node data | variable | Extra data for some kinds of nodes. |// | Structured data | variable | Msgpack-encoded metadata blobs (e.g. file references). |// | Nodes | 28 bytes per node | Defines the AST structure of the file, with references to strings and extended data. |//// Header (44 bytes)// -----------------//// The header contains the following fields://// | Byte offset | Type | Field |// | ----------- | --------- | ------------------------------------------------- |// | 0 | uint8 | Protocol version |// | 1-3 | | Reserved |// | 4-19 | uint128 | Source file content hash (xxh3, LE) |// | 20-23 | uint32 | Parse options (bitmask; bit 0: JSX, bit 1: Force) |// | 24-27 | uint32 | Byte offset to string offsets section |// | 28-31 | uint32 | Byte offset to string data section |// | 32-35 | uint32 | Byte offset to extended node data section |// | 36-39 | uint32 | Byte offset to structured data section |// | 40-43 | uint32 | Byte offset to nodes section |//// String offsets (8 bytes per string)// -----------------------------------//// Each string offset entry consists of two 4-byte unsigned integers, representing the start and end byte offsets into the// **string data** section.//// String data (variable)// ----------------------//// The string data section contains UTF-8 encoded string data. In typical cases, the entirety of the string data is the// source file text, and individual nodes with string properties reference their positional slice of the file text. In// cases where a node's string property is not equal to the slice of file text at its position, the unique string is// appended to the string data section after the file text.//// Extended node data (variable)// -----------------------------//// The extended node data section contains additional data for specific node types. The length and meaning of each entry// is defined by the node type.//// Currently, the only node types that use this section are `TemplateHead`, `TemplateMiddle`, `TemplateTail`, and// `SourceFile`. The extended data format for the first three is://// | Byte offset | Type | Field |// | ----------- | ------ | ------------------------------------------------ |// | 0-4 | uint32 | Index of `text` in the string offsets section |// | 4-8 | uint32 | Index of `rawText` in the string offsets section |// | 8-12 | uint32 | Value of `templateFlags` |//// and for `SourceFile` is://// | Byte offset | Type | Field |// | ----------- | ------ | -------------------------------------------------------------- |// | 0-4 | uint32 | Index of `text` in the string offsets section |// | 4-8 | uint32 | Index of `fileName` in the string offsets section |// | 8-12 | uint32 | Index of `path` in the string offsets section |// | 12-16 | uint32 | Value of `languageVariant` |// | 16-20 | uint32 | Value of `scriptKind` |// | 20-24 | uint32 | Byte offset of `referencedFiles` in structured data section |// | 24-28 | uint32 | Byte offset of `typeReferenceDirectives` in structured data |// | 28-32 | uint32 | Byte offset of `libReferenceDirectives` in structured data |// | 32-36 | uint32 | Byte offset of `imports` node index array in structured data |// | 36-40 | uint32 | Byte offset of `moduleAugmentations` node index array |// | 40-44 | uint32 | Byte offset of `ambientModuleNames` string array |// | 44-48 | uint32 | Node index of `externalModuleIndicator` (0 = nil) |//// Structured data (variable)// --------------------------//// The structured data section contains msgpack-encoded metadata blobs. Each blob is a self-contained// msgpack value. File reference arrays use the following tuple format://// [pos: uint, end: uint, fileName: string, resolutionMode: uint, preserve: bool]//// Node index arrays (imports, moduleAugmentations) are msgpack arrays of uint values, where each// value is a node index into the nodes section. String arrays (ambientModuleNames) are msgpack// arrays of string values.//// An offset of 0xFFFFFFFF indicates no data (empty array).//// Nodes (28 bytes per node)// -------------------------//// The nodes section contains the AST structure of the file. Nodes are represented in a flat array in source order,// heavily inspired by https://marvinh.dev/blog/speeding-up-javascript-ecosystem-part-11/. Each node has the following// structure://// | Byte offset | Type | Field |// | ----------- | ------ | -------------------------- |// | 0-4 | uint32 | Kind |// | 4-8 | uint32 | Pos |// | 8-12 | uint32 | End |// | 12-16 | uint32 | Node index of next sibling |// | 16-20 | uint32 | Node index of parent |// | 20-24 | | Node data |// | 24-28 | uint32 | Node flags |//// The first 28 bytes of the nodes section are zeros representing a nil node, such that nodes without a parent or next// sibling can unambiuously use `0` for those indices.//// NodeLists are represented as normal nodes with the special `kind` value `0xff_ff_ff_ff`. They are considered the parent// of their contents in the encoded format. A client reconstructing an AST similar to TypeScript's internal representation// should instead set the `parent` pointers of a NodeList's children to the NodeList's parent. A NodeList's `data` field// is the uint32 length of the list, and does not use one of the data types described below.//// For node types other than NodeList, the node data field encodes one of the following, determined by the first 2 bits of// the field://// | Value | Data type | Description |// | ----- | --------- | ------------------------------------------------------------------------------------ |// | 0b00 | Children | Disambiguates which named properties of the node its children should be assigned to. |// | 0b01 | String | The index of the node's string property in the **string offsets** section. |// | 0b10 | Extended | The byte offset of the node's extended data into the **extended node data** section. |// | 0b11 | Reserved | Reserved for future use. |//// In all node data types, the remaining 6 bits of the first byte are used to encode small values specific to the node// type. For most node types, these are individual boolean flags. For unary expressions, all 6 bits encode the operator's// SyntaxKind value (e.g., PlusPlusToken=45, TildeToken=54), which fits because KindLastUnaryOperator (54) <= 0x3f (63).//// | Node type | Bits 0-5 | Notes |// | ---------------------------- | ------------------------------------- | ------------------------------ |// | `ImportSpecifier` | Bit 0: `isTypeOnly` | |// | `ImportClause` | Bit 0: `isTypeOnly`, Bit 1: `isDefer` | |// | `ExportSpecifier` | Bit 0: `isTypeOnly` | |// | `ImportEqualsDeclaration` | Bit 0: `isTypeOnly` | |// | `ExportDeclaration` | Bit 0: `isTypeOnly` | |// | `ImportTypeNode` | Bit 0: `isTypeOf` | |// | `ExportAssignment` | Bit 0: `isExportEquals` | |// | `Block` | Bit 0: `multiline` | |// | `ArrayLiteralExpression` | Bit 0: `multiline` | |// | `ObjectLiteralExpression` | Bit 0: `multiline` | |// | `JsxText` | Bit 0: `containsOnlyTriviaWhiteSpaces`| |// | `JSDocTypeLiteral` | Bit 0: `isArrayType` | |// | `JsDocPropertyTag` | Bit 0: `isBracketed`, Bit 1: `isNameFirst` | |// | `JsDocParameterTag` | Bit 0: `isBracketed`, Bit 1: `isNameFirst` | |// | `VariableDeclarationList` | Bit 0: is `let`, Bit 1: is `const` | |// | `ImportAttributes` | Bit 0: `multiline`, Bit 1: is `assert`| |// | `PrefixUnaryExpression` | Bits 0-5: operator SyntaxKind | e.g., `!`, `~`, `++`, `--` |// | `PostfixUnaryExpression` | Bits 0-5: operator SyntaxKind | e.g., `++`, `--` |//// The remaining 3 bytes of the node data field vary by data type://// ### Children (0b00)//// If a node has fewer children than its type allows, additional data is needed to determine which properties the children// correspond to. The last byte of the 4-byte data field is a bitmask representing the child properties of the node type,// in visitor order, where `1` indicates that the child at that property is present and `0` indicates that the property is// nil. For example, a `MethodDeclaration` has the following child properties://// | Property name | Bit position |// | -------------- | ------------ |// | modifiers | 0 |// | asteriskToken | 1 |// | name | 2 |// | postfixToken | 3 |// | typeParameters | 4 |// | parameters | 5 |// | returnType | 6 |// | body | 7 |//// A bitmask with value `0b01100101` would indicate that the next four direct descendants (i.e., node records that have a// `parent` set to the node index of the `MethodDeclaration`) of the node are its `modifiers`, `name`, `parameters`, and// `body` properties, in that order. The remaining properties are nil. (To reconstruct the node with named properties, the// client must consult a static table of each node type's child property names.)//// The bitmask may be zero for node types that can only have a single child, since no disambiguation is needed.// Additionally, the children data type may be used for nodes that can never have children, but do not require other// data types.//// ### String (0b01)//// The string data type is used for nodes with a single string property. (Currently, the name of that property is always// `text`.) The last three bytes of the 4-byte data field form a single 24-bit unsigned integer (i.e.,// `uint32(0x00_ff_ff_ff & node.data)`) _N_ that is an index into the **string offsets** section. The *N*th 32-bit// unsigned integer in the **string offsets** section is the byte offset of the start of the string in the **string data**// section, and the *N+1*th 32-bit unsigned integer is the byte offset of the end of the string in the// **string data** section.//// ### Extended (0b10)//// The extended data type is used for nodes with properties that don't fit into either the children or string data types.// The last three bytes of the 4-byte data field form a single 24-bit unsigned integer (i.e.,// `uint32(0x00_ff_ff_ff & node.data)`) _N_ that is a byte offset into the **extended node data** section. The length and// meaning of the data at that offset is defined by the node type. See the **Extended node data** section for details on// the format of the extended data for specific node types.//// Encoding Arbitrary Nodes// ------------------------//// The same binary format can be used to encode an arbitrary subtree of a SourceFile, not just a whole SourceFile. When// encoding a non-SourceFile node, the format is identical with the following differences://// - The content hash fields in the header (bytes 4-19) are zero.// - The parse options field in the header (bytes 20-23) is zero.// - The root node in the nodes section uses its actual node kind and data encoding (via getNodeData) rather than the// SourceFile-specific extended data format.//// The string data section contains only the strings referenced by nodes in the subtree, rather than the full source// file text. The EncodeNode function provides this entrypoint.
// SourceFileHash returns the 128-bit content hash for a source file as a hex string.func SourceFileHash(sourceFile *ast.SourceFile) string { h := sourceFile.Hash return fmt.Sprintf("%016x%016x", h.Hi, h.Lo)}
// encodeParseOptions encodes the per-file ExternalModuleIndicatorOptions as a uint32 bitmask.func encodeParseOptions(opts ast.ExternalModuleIndicatorOptions) uint32 { var bits uint32 if opts.JSX { bits |= 1 } if opts.Force { bits |= 2 } return bits}
// NodeIndexTable maps between AST nodes and their encoder indices for O(1) node handle resolution.type NodeIndexTable struct { Nodes []*ast.Node // index → node (for resolution) sortedOnce sync.Once sortedIdx []uint32 // indices into Nodes, sorted by node ID; built lazily}
// GetIndex returns the encoder index for the given node.// On the first call the sortedIdx array is built (O(n log n) sort on a flat []uint32),// then subsequent calls use binary search (O(log n)). This turns out to be much faster than// building a map[*ast.Node]uint32 and not significantly slower for lookups.func (t *NodeIndexTable) GetIndex(node *ast.Node) uint32 { t.sortedOnce.Do(func() { idx := make([]uint32, 0, len(t.Nodes)) for i, n := range t.Nodes { if n != nil { idx = append(idx, uint32(i)) } } nodes := t.Nodes slices.SortFunc(idx, func(a, b uint32) int { return cmp.Compare(ast.GetNodeId(nodes[a]), ast.GetNodeId(nodes[b])) }) t.sortedIdx = idx }) target := ast.GetNodeId(node) i, found := core.BinarySearchUniqueFunc(t.sortedIdx, func(_ int, el uint32) int { return cmp.Compare(ast.GetNodeId(t.Nodes[el]), target) }) if found { return t.sortedIdx[i] } return 0}
// BuildNodeIndexTable walks the AST in the same order as encodeTree and builds// a NodeIndexTable without performing the full binary encoding. This is used to// eagerly create index tables for files that need node handles before getSourceFile// is called. The indices produced are guaranteed to match those from EncodeSourceFile.func BuildNodeIndexTable(sourceFile *ast.SourceFile) *NodeIndexTable { var nodeCount uint32 nodeTable := make([]*ast.Node, 1, sourceFile.NodeCount+1) // index 0 = nil sentinel
visitor := &ast.NodeVisitor{ Hooks: ast.NodeVisitorHooks{ VisitNodes: func(nodeList *ast.NodeList, visitor *ast.NodeVisitor) *ast.NodeList { if nodeList == nil { return nodeList } nodeCount++ nodeTable = append(nodeTable, nil) // NodeLists are not *ast.Node visitor.VisitSlice(nodeList.Nodes) return nodeList }, VisitModifiers: func(modifiers *ast.ModifierList, visitor *ast.NodeVisitor) *ast.ModifierList { if modifiers != nil && len(modifiers.Nodes) > 0 { visitor.Hooks.VisitNodes(&modifiers.NodeList, visitor) } return modifiers }, }, } visitor.Visit = func(node *ast.Node) *ast.Node { nodeCount++ nodeTable = append(nodeTable, node) visitor.VisitEachChild(node) for _, jsdoc := range node.JSDoc(sourceFile) { visitor.Visit(jsdoc) } return node }
rootNode := sourceFile.AsNode() // Index 1 = root node (matches encodeTree) nodeCount++ nodeTable = append(nodeTable, rootNode)
visitor.VisitEachChild(rootNode) for _, jsdoc := range rootNode.JSDoc(sourceFile) { visitor.Visit(jsdoc) }
return &NodeIndexTable{Nodes: nodeTable}}
// EncodeSourceFile encodes an entire source file AST into the binary format.// Returns the encoded bytes and a NodeIndexTable mapping encoder indices to AST nodes.func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, error) { return encodeTree(sourceFile.AsNode(), sourceFile)}
// EncodeNode encodes an arbitrary AST node and its descendants into the binary format.// The sourceFile is needed to provide the source text for efficient string encoding.// When encoding a non-SourceFile node, the header hash and parse options fields will be zero.// Returns the encoded bytes and a NodeIndexTable mapping encoder indices to AST nodes.func EncodeNode(node *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, error) { return encodeTree(node, sourceFile)}
func encodeTree(rootNode *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, error) { var parentIndex, nodeCount, prevIndex uint32 var extendedData []byte var structuredData []byte var strs *stringTable var positionMap *ast.PositionMap if rootNode.Kind == ast.KindSourceFile { strs = newStringTable(sourceFile.Text(), sourceFile.TextCount) positionMap = sourceFile.GetPositionMap() } else { strs = newStringTable("", 0) if sourceFile != nil { positionMap = sourceFile.GetPositionMap() } } if positionMap == nil { positionMap = ast.ComputePositionMap("") } utf16 := func(pos int) uint32 { return uint32(positionMap.UTF8ToUTF16(pos)) } var initialNodeCount int if sourceFile != nil { initialNodeCount = sourceFile.NodeCount } nodes := make([]byte, 0, (initialNodeCount+1)*NodeSize)
// Build node index table for O(1) handle resolution. // Index 0 is a nil sentinel; real nodes start at index 1. nodeTable := make([]*ast.Node, 1, initialNodeCount+1) // index 0 = nil sentinel
// Build a small map of nodes we need to track indices for (imports + moduleAugmentations). // Values start at 0 and are filled in during the walk. var nodeIndexMap map[*ast.Node]uint32 var sfExtendedDataOffset int // byte offset in extendedData where SourceFile fields start if rootNode.Kind == ast.KindSourceFile { sf := rootNode.AsSourceFile() total := len(sf.Imports()) + len(sf.ModuleAugmentations) if sf.ExternalModuleIndicator != nil && sf.ExternalModuleIndicator != rootNode { total++ } if total > 0 { nodeIndexMap = make(map[*ast.Node]uint32, total) for _, imp := range sf.Imports() { nodeIndexMap[imp.AsNode()] = 0 } for _, aug := range sf.ModuleAugmentations { nodeIndexMap[aug.AsNode()] = 0 } if sf.ExternalModuleIndicator != nil && sf.ExternalModuleIndicator != rootNode { nodeIndexMap[sf.ExternalModuleIndicator] = 0 } } }
visitor := &ast.NodeVisitor{ Hooks: ast.NodeVisitorHooks{ VisitNodes: func(nodeList *ast.NodeList, visitor *ast.NodeVisitor) *ast.NodeList { if nodeList == nil { return nodeList }
nodeCount++ nodeTable = append(nodeTable, nil) // NodeLists are not *ast.Node if prevIndex != 0 { // this is the next sibling of `prevNode` b0, b1, b2, b3 := uint8(nodeCount), uint8(nodeCount>>8), uint8(nodeCount>>16), uint8(nodeCount>>24) nodes[prevIndex*NodeSize+NodeOffsetNext+0] = b0 nodes[prevIndex*NodeSize+NodeOffsetNext+1] = b1 nodes[prevIndex*NodeSize+NodeOffsetNext+2] = b2 nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3 }
nodes = appendUint32s(nodes, SyntaxKindNodeList, utf16(nodeList.Pos()), utf16(nodeList.End()), 0, parentIndex, uint32(len(nodeList.Nodes)), 0)
saveParentIndex := parentIndex
currentIndex := nodeCount prevIndex = 0 parentIndex = currentIndex visitor.VisitSlice(nodeList.Nodes) prevIndex = currentIndex parentIndex = saveParentIndex
return nodeList }, VisitModifiers: func(modifiers *ast.ModifierList, visitor *ast.NodeVisitor) *ast.ModifierList { if modifiers != nil && len(modifiers.Nodes) > 0 { visitor.Hooks.VisitNodes(&modifiers.NodeList, visitor) } return modifiers }, }, } visitor.Visit = func(node *ast.Node) *ast.Node { nodeCount++ nodeTable = append(nodeTable, node) if prevIndex != 0 { // this is the next sibling of `prevNode` b0, b1, b2, b3 := uint8(nodeCount), uint8(nodeCount>>8), uint8(nodeCount>>16), uint8(nodeCount>>24) nodes[prevIndex*NodeSize+NodeOffsetNext+0] = b0 nodes[prevIndex*NodeSize+NodeOffsetNext+1] = b1 nodes[prevIndex*NodeSize+NodeOffsetNext+2] = b2 nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3 }
nodes = appendUint32s(nodes, uint32(node.Kind), utf16(node.Pos()), utf16(node.End()), 0, parentIndex, getNodeData(node, strs, positionMap, &extendedData, &structuredData), uint32(node.Flags))
if nodeIndexMap != nil { if _, ok := nodeIndexMap[node]; ok { nodeIndexMap[node] = nodeCount } }
saveParentIndex := parentIndex
currentIndex := nodeCount prevIndex = 0 parentIndex = currentIndex visitor.VisitEachChild(node) if sourceFile != nil { for _, jsdoc := range node.JSDoc(sourceFile) { visitor.Visit(jsdoc) } } prevIndex = currentIndex parentIndex = saveParentIndex return node }
nodes = appendUint32s(nodes, 0, 0, 0, 0, 0, 0, 0)
nodeCount++ parentIndex++ nodeTable = append(nodeTable, rootNode) // index 1 = root node
sfExtendedDataOffset = len(extendedData) nodes = appendUint32s(nodes, uint32(rootNode.Kind), utf16(rootNode.Pos()), utf16(rootNode.End()), 0, 0, getNodeData(rootNode, strs, positionMap, &extendedData, &structuredData), uint32(rootNode.Flags))
visitor.VisitEachChild(rootNode) if sourceFile != nil { for _, jsdoc := range rootNode.JSDoc(sourceFile) { visitor.Visit(jsdoc) } }
var hash xxh3.Uint128 var parseOpts uint32 if rootNode.Kind == ast.KindSourceFile { hash = sourceFile.Hash parseOpts = encodeParseOptions(sourceFile.ParseOptions().ExternalModuleIndicatorOptions)
// Encode imports, moduleAugmentations, and ambientModuleNames into structured data, // and patch the placeholder offsets in the SourceFile extended data. sf := rootNode.AsSourceFile() importsOffset := encodeNodeIndexArray(sf.Imports(), nodeIndexMap, &structuredData) moduleAugmentationsOffset := encodeModuleAugmentations(sf.ModuleAugmentations, nodeIndexMap, &structuredData) ambientModuleNamesOffset := encodeStringArray(sf.AmbientModuleNames, &structuredData) // Patch the 3 placeholder uint32s at sfExtendedDataOffset + 32, 36, 40 binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+32:], importsOffset) binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+36:], moduleAugmentationsOffset) binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+40:], ambientModuleNamesOffset) // Patch externalModuleIndicator node index at offset 44 var externalModuleIndicatorIndex uint32 if sf.ExternalModuleIndicator != nil { if sf.ExternalModuleIndicator == rootNode { externalModuleIndicatorIndex = 1 // root node index } else { externalModuleIndicatorIndex = nodeIndexMap[sf.ExternalModuleIndicator] } } binary.LittleEndian.PutUint32(extendedData[sfExtendedDataOffset+44:], externalModuleIndicatorIndex) }
metadata := uint32(ProtocolVersion) << 24 offsetStringTableOffsets := HeaderSize offsetStringTableData := HeaderSize + len(strs.offsets)*4 offsetExtendedData := offsetStringTableData + strs.stringLength() offsetStructuredData := offsetExtendedData + len(extendedData) offsetNodes := offsetStructuredData + len(structuredData)
header := []uint32{ metadata, uint32(hash.Lo), uint32(hash.Lo >> 32), uint32(hash.Hi), uint32(hash.Hi >> 32), parseOpts, uint32(offsetStringTableOffsets), uint32(offsetStringTableData), uint32(offsetExtendedData), uint32(offsetStructuredData), uint32(offsetNodes), }
var headerBytes, strsBytes []byte headerBytes = appendUint32s(nil, header...) strsBytes = strs.encode()
return slices.Concat( headerBytes, strsBytes, extendedData, structuredData, nodes, ), &NodeIndexTable{Nodes: nodeTable}, nil}
func appendUint32s(buf []byte, values ...uint32) []byte { for _, value := range values { buf = binary.LittleEndian.AppendUint32(buf, value) } return buf}
func getNodeData(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) uint32 { t := getNodeDataType(node) switch t { case NodeDataTypeChildren: return t | getNodeCommonData(node) | uint32(getChildrenPropertyMask(node)) case NodeDataTypeString: return t | getNodeCommonData(node) | recordNodeStrings(node, strs) case NodeDataTypeExtendedData: return t | getNodeCommonData(node) | recordExtendedData(node, strs, positionMap, extendedData, structuredData) default: panic("unreachable") }}
const noStructuredData = 0xFFFFFFFF
func recordExtendedData_SourceFile(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) { sf := node.AsSourceFile() textIndex := strs.add(sf.Text(), sf.Kind, sf.Pos(), sf.End()) fileNameIndex := strs.add(sf.FileName(), 0, 0, 0) pathIndex := strs.add(string(sf.Path()), 0, 0, 0) referencedFilesOffset := encodeFileReferences(sf.ReferencedFiles, positionMap, structuredData) typeRefDirectivesOffset := encodeFileReferences(sf.TypeReferenceDirectives, positionMap, structuredData) libRefDirectivesOffset := encodeFileReferences(sf.LibReferenceDirectives, positionMap, structuredData) // imports, moduleAugmentations, ambientModuleNames offsets are placeholders; // they will be patched after the tree walk when node indices are known. *extendedData = appendUint32s(*extendedData, textIndex, fileNameIndex, pathIndex, uint32(sf.LanguageVariant), uint32(sf.ScriptKind), referencedFilesOffset, typeRefDirectivesOffset, libRefDirectivesOffset, noStructuredData, noStructuredData, noStructuredData, 0)}
func recordExtendedData_TemplateHead(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) { n := node.AsTemplateHead() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))}
func recordExtendedData_TemplateMiddle(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) { n := node.AsTemplateMiddle() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))}
func recordExtendedData_TemplateTail(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) { n := node.AsTemplateTail() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))}
func boolToByte(b bool) byte { if b { return 1 } return 0}
// hasModifiers returns true if the modifier list is non-nil and has at least one modifier.func hasModifiers(modifiers *ast.ModifierList) bool { return modifiers != nil && len(modifiers.Nodes) > 0}
// encodeFileReferences encodes a slice of FileReferences as a msgpack array of tuples// into the structured data buffer. Returns the byte offset into the buffer, or// noStructuredData (0xFFFFFFFF) if the slice is empty.func encodeFileReferences(refs []*ast.FileReference, positionMap *ast.PositionMap, buf *[]byte) uint32 { if len(refs) == 0 { return noStructuredData } offset := uint32(len(*buf)) *buf = msgpackWriteArrayHeader(*buf, len(refs)) for _, ref := range refs { // Each entry is a 5-element tuple: [pos, end, fileName, resolutionMode, preserve] *buf = msgpackWriteArrayHeader(*buf, 5) *buf = msgpackWriteUint(*buf, uint32(positionMap.UTF8ToUTF16(ref.Pos()))) *buf = msgpackWriteUint(*buf, uint32(positionMap.UTF8ToUTF16(ref.End()))) *buf = msgpackWriteString(*buf, ref.FileName) *buf = msgpackWriteUint(*buf, uint32(ref.ResolutionMode)) *buf = msgpackWriteBool(*buf, ref.Preserve) } return offset}
// encodeNodeIndexArray encodes a slice of LiteralLikeNodes as a msgpack array of// uint node indices. Returns the byte offset into the buffer, or noStructuredData// if the slice is empty.func encodeNodeIndexArray(nodes []*ast.LiteralLikeNode, indexMap map[*ast.Node]uint32, buf *[]byte) uint32 { if len(nodes) == 0 { return noStructuredData } offset := uint32(len(*buf)) *buf = msgpackWriteArrayHeader(*buf, len(nodes)) for _, node := range nodes { *buf = msgpackWriteUint(*buf, indexMap[node.AsNode()]) } return offset}
// encodeModuleAugmentations encodes a slice of ModuleName nodes as a msgpack array// of uint node indices. Returns the byte offset into the buffer, or noStructuredData// if the slice is empty.func encodeModuleAugmentations(nodes []*ast.ModuleName, indexMap map[*ast.Node]uint32, buf *[]byte) uint32 { if len(nodes) == 0 { return noStructuredData } offset := uint32(len(*buf)) *buf = msgpackWriteArrayHeader(*buf, len(nodes)) for _, node := range nodes { *buf = msgpackWriteUint(*buf, indexMap[node.AsNode()]) } return offset}
// encodeStringArray encodes a slice of strings as a msgpack array of strings.// Returns the byte offset into the buffer, or noStructuredData if the slice is empty.func encodeStringArray(strs []string, buf *[]byte) uint32 { if len(strs) == 0 { return noStructuredData } offset := uint32(len(*buf)) *buf = msgpackWriteArrayHeader(*buf, len(strs)) for _, s := range strs { *buf = msgpackWriteString(*buf, s) } return offset}
// Minimal msgpack writers for the structured data section.
func msgpackWriteArrayHeader(buf []byte, length int) []byte { if length <= 0x0f { return append(buf, byte(0x90|length)) } if length <= 0xffff { return append(buf, 0xdc, byte(length>>8), byte(length)) } return append(buf, 0xdd, byte(length>>24), byte(length>>16), byte(length>>8), byte(length))}
func msgpackWriteUint(buf []byte, value uint32) []byte { if value <= 0x7f { return append(buf, byte(value)) } if value <= 0xff { return append(buf, 0xcc, byte(value)) } if value <= 0xffff { return append(buf, 0xcd, byte(value>>8), byte(value)) } return append(buf, 0xce, byte(value>>24), byte(value>>16), byte(value>>8), byte(value))}
func msgpackWriteString(buf []byte, s string) []byte { n := len(s) if n <= 0x1f { buf = append(buf, byte(0xa0|n)) } else if n <= 0xff { buf = append(buf, 0xd9, byte(n)) } else if n <= 0xffff { buf = append(buf, 0xda, byte(n>>8), byte(n)) } else { buf = append(buf, 0xdb, byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) } return append(buf, s...)}
func msgpackWriteBool(buf []byte, value bool) []byte { if value { return append(buf, 0xc3) } return append(buf, 0xc2)}
// Hand-written commonData encoding functions for nodes whose non-bool data// members cannot be automatically encoded by the generator. Each function// packs relevant fields into the 6-bit commonData area (bits 24-29) of the// 32-bit node data word.
func getNodeCommonData_SyntheticExpression(_ *ast.Node) uint32 { // SyntheticExpression is an internal compiler node that is never part of a parsed AST. // It should never be encoded. panic("SyntheticExpression should never be encoded")}
// Hand-written extended data encoding functions for literal nodes that were// previously string-type but whose TokenFlags/TemplateFlags cannot fit in 6 bits.
func recordExtendedData_StringLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) { n := node.AsStringLiteral() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))}
func recordExtendedData_NumericLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) { n := node.AsNumericLiteral() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))}
func recordExtendedData_BigIntLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) { n := node.AsBigIntLiteral() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))}
func recordExtendedData_RegularExpressionLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) { n := node.AsRegularExpressionLiteral() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TokenFlags))}
func recordExtendedData_NoSubstitutionTemplateLiteral(node *ast.Node, strs *stringTable, _ *ast.PositionMap, extendedData *[]byte, _ *[]byte) { n := node.AsNoSubstitutionTemplateLiteral() textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End()) *extendedData = appendUint32s(*extendedData, textIndex, uint32(n.TemplateFlags))}