// Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. package token // Flags are the low 8 bits of an ID. For example, they signify whether a token // is an operator, an identifier, etc. // // The flags are not exclusive. For example, the "+" token is both a unary and // binary operator. It is also associative: (a + b) + c equals a + (b + c). // // A valid token will have non-zero flags. If none of the other flags apply, // the FlagsOther bit will be set. type Flags uint32 const ( FlagsOther = Flags(0x01) FlagsUnaryOp = Flags(0x02) FlagsBinaryOp = Flags(0x04) FlagsAssociative = Flags(0x08) FlagsAssign = Flags(0x10) FlagsLiteral = Flags(0x20) FlagsIdent = Flags(0x40) FlagsImplicitSemicolon = Flags(0x80) ) // Key is the high 24 bits of an ID. It is the map key for an IDMap. type Key uint32 const ( idShift = 8 idMask = 1<<8 - 1 maxKey = 1<<24 - 1 ) // ID combines a Key and Flags. type ID uint32 func (x ID) Key() Key { return Key(x >> idShift) } func (x ID) Flags() Flags { return Flags(x & idMask) } func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } func (x ID) IsAssociative() bool { return Flags(x)&FlagsAssociative != 0 } func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 } func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 } func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 } // nBuiltInKeys is the number of built-in Keys. The packing is: // - Zero is invalid. // - [0x01, 0x3F] are squiggly punctuation, such as "(", ")" and ";". // - [0x40, 0x5F] are squiggly assignments, such as "=" and "+=". // - [0x60, 0x7F] are squiggly operators, such as "+" and "==". // - [0x80, 0x9F] are alpha-numeric operators, such as "and" and "as". // - [0xA0, 0xCF] are keywords, such as "if" and "return". // - [0xD0, 0xDF] are literals, such as "false" and "true". // - [0xE0, 0xFF] are identifiers, such as "bool" and "u32". // // "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and // "&=". Their Keys range in [0x01, 0x7F]. const nBuiltInKeys = 256 const ( IDInvalid = ID(0) IDOpenParen = ID(0x10<> idShift: {"(", IDOpenParen}, IDCloseParen >> idShift: {")", IDCloseParen}, IDOpenBracket >> idShift: {"[", IDOpenBracket}, IDCloseBracket >> idShift: {"]", IDCloseBracket}, IDOpenCurly >> idShift: {"{", IDOpenCurly}, IDCloseCurly >> idShift: {"}", IDCloseCurly}, IDDot >> idShift: {".", IDDot}, IDComma >> idShift: {",", IDComma}, IDQuestion >> idShift: {"?", IDQuestion}, IDColon >> idShift: {":", IDColon}, IDSemicolon >> idShift: {";", IDSemicolon}, IDEq >> idShift: {"=", IDEq}, IDPlusEq >> idShift: {"+=", IDPlusEq}, IDMinusEq >> idShift: {"-=", IDMinusEq}, IDStarEq >> idShift: {"*=", IDStarEq}, IDSlashEq >> idShift: {"/=", IDSlashEq}, IDShiftLEq >> idShift: {"<<=", IDShiftLEq}, IDShiftREq >> idShift: {">>=", IDShiftREq}, IDAmpEq >> idShift: {"&=", IDAmpEq}, IDAmpHatEq >> idShift: {"&^=", IDAmpHatEq}, IDPipeEq >> idShift: {"|=", IDPipeEq}, IDHatEq >> idShift: {"^=", IDHatEq}, IDPlus >> idShift: {"+", IDPlus}, IDMinus >> idShift: {"-", IDMinus}, IDStar >> idShift: {"*", IDStar}, IDSlash >> idShift: {"/", IDSlash}, IDShiftL >> idShift: {"<<", IDShiftL}, IDShiftR >> idShift: {">>", IDShiftR}, IDAmp >> idShift: {"&", IDAmp}, IDAmpHat >> idShift: {"&^", IDAmpHat}, IDPipe >> idShift: {"|", IDPipe}, IDHat >> idShift: {"^", IDHat}, IDNotEq >> idShift: {"!=", IDNotEq}, IDLessThan >> idShift: {"<", IDLessThan}, IDLessEq >> idShift: {"<=", IDLessEq}, IDEqEq >> idShift: {"==", IDEqEq}, IDGreaterEq >> idShift: {">=", IDGreaterEq}, IDGreaterThan >> idShift: {">", IDGreaterThan}, IDAnd >> idShift: {"and", IDAnd}, IDOr >> idShift: {"or", IDOr}, IDNot >> idShift: {"not", IDNot}, IDAs >> idShift: {"as", IDAs}, IDFunc >> idShift: {"func", IDFunc}, IDFalse >> idShift: {"false", IDFalse}, IDTrue >> idShift: {"true", IDTrue}, IDI8 >> idShift: {"i8", IDI8}, IDI16 >> idShift: {"i16", IDI16}, IDI32 >> idShift: {"i32", IDI32}, IDI64 >> idShift: {"i64", IDI64}, IDU8 >> idShift: {"u8", IDU8}, IDU16 >> idShift: {"u16", IDU16}, IDU32 >> idShift: {"u32", IDU32}, IDU64 >> idShift: {"u64", IDU64}, IDUsize >> idShift: {"usize", IDUsize}, IDBool >> idShift: {"bool", IDBool}, IDBuf1 >> idShift: {"buf1", IDBuf1}, IDBuf2 >> idShift: {"buf2", IDBuf2}, IDUnderscore >> idShift: {"_", IDUnderscore}, IDThis >> idShift: {"this", IDThis}, } var builtInsByName = map[string]ID{} func init() { for _, x := range builtInsByKey { if x.name != "" { builtInsByName[x.name] = x.id } } } // squiggles are built-in IDs that aren't alpha-numeric. var squiggles = [256]ID{ '(': IDOpenParen, ')': IDCloseParen, '[': IDOpenBracket, ']': IDCloseBracket, '{': IDOpenCurly, '}': IDCloseCurly, '.': IDDot, ',': IDComma, '?': IDQuestion, ':': IDColon, ';': IDSemicolon, // 1<': 1 << idShift, } type suffixLexer struct { suffix string id ID } // lexers lex ambiguous 1-byte squiggles. For example, "&" might be the start // of "&^" or "&=". // // The order of the []suffixLexer elements matters. The first match wins. Since // we want to lex greedily, longer suffixes should be earlier in the slice. var lexers = [256][]suffixLexer{ '!': { {"=", IDNotEq}, }, '&': { {"^=", IDAmpHatEq}, {"^", IDAmpHat}, {"=", IDAmpEq}, {"", IDAmp}, }, '|': { {"=", IDPipeEq}, {"", IDPipe}, }, '^': { {"=", IDHatEq}, {"", IDHat}, }, '+': { {"=", IDPlusEq}, {"", IDPlus}, }, '-': { {"=", IDMinusEq}, {"", IDMinus}, }, '*': { {"=", IDStarEq}, {"", IDStar}, }, '/': { {"=", IDSlashEq}, {"", IDSlash}, }, '=': { {"=", IDEqEq}, {"", IDEq}, }, '<': { {"<=", IDShiftLEq}, {"<", IDShiftL}, {"=", IDLessEq}, {"", IDLessThan}, }, '>': { {">=", IDShiftREq}, {">", IDShiftR}, {"=", IDGreaterEq}, {"", IDGreaterThan}, }, }