// Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. package token // Flags are the low 16 bits of an ID. For example, they signify whether a // token is an operator, an identifier, etc. // // The flags are not exclusive. For example, the "+" token is both a unary and // binary operator. It is also associative: (a + b) + c equals a + (b + c). // // A valid token will have non-zero flags. If none of the other flags apply, // the FlagsOther bit will be set. type Flags uint32 const ( FlagsOther = Flags(0x0001) FlagsUnaryOp = Flags(0x0002) FlagsBinaryOp = Flags(0x0004) FlagsAssociativeOp = Flags(0x0008) FlagsLiteral = Flags(0x0010) FlagsNumLiteral = Flags(0x0020) FlagsStrLiteral = Flags(0x0040) FlagsIdent = Flags(0x0080) FlagsOpen = Flags(0x0100) FlagsClose = Flags(0x0200) FlagsTightLeft = Flags(0x0400) FlagsTightRight = Flags(0x0800) FlagsAssign = Flags(0x1000) FlagsImplicitSemicolon = Flags(0x2000) FlagsNumType = Flags(0x4000) flagsUnused = Flags(0x8000) ) // Key is the high 16 bits of an ID. It is the map key for an IDMap. type Key uint32 const ( FlagsBits = 16 FlagsMask = 1<>KeyShift)] } func (x ID) BinaryForm() ID { return binaryForms[0xFF&(x>>KeyShift)] } func (x ID) AssociativeForm() ID { return associativeForms[0xFF&(x>>KeyShift)] } func (x ID) Key() Key { return Key(x >> KeyShift) } func (x ID) Flags() Flags { return Flags(x & FlagsMask) } func (x ID) IsBuiltIn() bool { return Key(x>>KeyShift) < nBuiltInKeys } func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } func (x ID) IsAssociativeOp() bool { return Flags(x)&FlagsAssociativeOp != 0 } func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 } func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 } func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 } func (x ID) IsOpen() bool { return Flags(x)&FlagsOpen != 0 } func (x ID) IsClose() bool { return Flags(x)&FlagsClose != 0 } func (x ID) IsTightLeft() bool { return Flags(x)&FlagsTightLeft != 0 } func (x ID) IsTightRight() bool { return Flags(x)&FlagsTightRight != 0 } func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 } func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 } func (x ID) IsNumType() bool { return Flags(x)&FlagsNumType != 0 } // QID is a qualified ID, such as "foo.bar". QID[0] is "foo"'s ID and QID[1] is // "bar"'s. QID[0] may be 0 for a plain "bar". type QID [2]ID // Token combines an ID and the line number it was seen. type Token struct { ID ID Line uint32 } func (t Token) Key() Key { return Key(t.ID >> KeyShift) } func (t Token) Flags() Flags { return Flags(t.ID & FlagsMask) } func (t Token) IsBuiltIn() bool { return Key(t.ID>>KeyShift) < nBuiltInKeys } func (t Token) IsUnaryOp() bool { return Flags(t.ID)&FlagsUnaryOp != 0 } func (t Token) IsBinaryOp() bool { return Flags(t.ID)&FlagsBinaryOp != 0 } func (t Token) IsAssociativeOp() bool { return Flags(t.ID)&FlagsAssociativeOp != 0 } func (t Token) IsLiteral() bool { return Flags(t.ID)&FlagsLiteral != 0 } func (t Token) IsNumLiteral() bool { return Flags(t.ID)&FlagsNumLiteral != 0 } func (t Token) IsStrLiteral() bool { return Flags(t.ID)&FlagsStrLiteral != 0 } func (t Token) IsIdent() bool { return Flags(t.ID)&FlagsIdent != 0 } func (t Token) IsOpen() bool { return Flags(t.ID)&FlagsOpen != 0 } func (t Token) IsClose() bool { return Flags(t.ID)&FlagsClose != 0 } func (t Token) IsTightLeft() bool { return Flags(t.ID)&FlagsTightLeft != 0 } func (t Token) IsTightRight() bool { return Flags(t.ID)&FlagsTightRight != 0 } func (t Token) IsAssign() bool { return Flags(t.ID)&FlagsAssign != 0 } func (t Token) IsImplicitSemicolon() bool { return Flags(t.ID)&FlagsImplicitSemicolon != 0 } func (t Token) IsNumType() bool { return Flags(t.ID)&FlagsNumType != 0 } // nBuiltInKeys is the number of built-in Keys. The packing is: // - Zero is invalid. // - [0x01, 0x2F] are squiggly punctuation, such as "(", ")" and ";". // - [0x30, 0x3F] are squiggly assignments, such as "=" and "+=". // - [0x40, 0x5F] are operators, such as "+", "==" and "not". // - [0x60, 0x8F] are keywords, such as "if" and "return". // - [0x90, 0x9F] are literals, such as "false" and "true". // - [0xA0, 0xBF] are identifiers, such as "bool" and "u32". // - [0xC0, 0xFF] are disambiguation forms, e.g. unary "+" vs binary "+". // // "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and // "&=". Their Keys range in [0x01, 0x7F]. const nBuiltInKeys = Key(256) const ( IDInvalid = ID(0) IDOpenParen = ID(0x10<> KeyShift: {"(", IDOpenParen}, IDCloseParen >> KeyShift: {")", IDCloseParen}, IDOpenBracket >> KeyShift: {"[", IDOpenBracket}, IDCloseBracket >> KeyShift: {"]", IDCloseBracket}, IDOpenCurly >> KeyShift: {"{", IDOpenCurly}, IDCloseCurly >> KeyShift: {"}", IDCloseCurly}, IDDot >> KeyShift: {".", IDDot}, IDComma >> KeyShift: {",", IDComma}, IDQuestion >> KeyShift: {"?", IDQuestion}, IDColon >> KeyShift: {":", IDColon}, IDSemicolon >> KeyShift: {";", IDSemicolon}, IDEq >> KeyShift: {"=", IDEq}, IDPlusEq >> KeyShift: {"+=", IDPlusEq}, IDMinusEq >> KeyShift: {"-=", IDMinusEq}, IDStarEq >> KeyShift: {"*=", IDStarEq}, IDSlashEq >> KeyShift: {"/=", IDSlashEq}, IDShiftLEq >> KeyShift: {"<<=", IDShiftLEq}, IDShiftREq >> KeyShift: {">>=", IDShiftREq}, IDAmpEq >> KeyShift: {"&=", IDAmpEq}, IDAmpHatEq >> KeyShift: {"&^=", IDAmpHatEq}, IDPipeEq >> KeyShift: {"|=", IDPipeEq}, IDHatEq >> KeyShift: {"^=", IDHatEq}, IDPlus >> KeyShift: {"+", IDPlus}, IDMinus >> KeyShift: {"-", IDMinus}, IDStar >> KeyShift: {"*", IDStar}, IDSlash >> KeyShift: {"/", IDSlash}, IDShiftL >> KeyShift: {"<<", IDShiftL}, IDShiftR >> KeyShift: {">>", IDShiftR}, IDAmp >> KeyShift: {"&", IDAmp}, IDAmpHat >> KeyShift: {"&^", IDAmpHat}, IDPipe >> KeyShift: {"|", IDPipe}, IDHat >> KeyShift: {"^", IDHat}, IDNotEq >> KeyShift: {"!=", IDNotEq}, IDLessThan >> KeyShift: {"<", IDLessThan}, IDLessEq >> KeyShift: {"<=", IDLessEq}, IDEqEq >> KeyShift: {"==", IDEqEq}, IDGreaterEq >> KeyShift: {">=", IDGreaterEq}, IDGreaterThan >> KeyShift: {">", IDGreaterThan}, IDAnd >> KeyShift: {"and", IDAnd}, IDOr >> KeyShift: {"or", IDOr}, IDNot >> KeyShift: {"not", IDNot}, IDAs >> KeyShift: {"as", IDAs}, IDFunc >> KeyShift: {"func", IDFunc}, IDPtr >> KeyShift: {"ptr", IDPtr}, IDAssert >> KeyShift: {"assert", IDAssert}, IDFor >> KeyShift: {"for", IDFor}, IDIf >> KeyShift: {"if", IDIf}, IDElse >> KeyShift: {"else", IDElse}, IDReturn >> KeyShift: {"return", IDReturn}, IDBreak >> KeyShift: {"break", IDBreak}, IDContinue >> KeyShift: {"continue", IDContinue}, IDStruct >> KeyShift: {"struct", IDStruct}, IDUse >> KeyShift: {"use", IDUse}, IDFalse >> KeyShift: {"false", IDFalse}, IDTrue >> KeyShift: {"true", IDTrue}, IDI8 >> KeyShift: {"i8", IDI8}, IDI16 >> KeyShift: {"i16", IDI16}, IDI32 >> KeyShift: {"i32", IDI32}, IDI64 >> KeyShift: {"i64", IDI64}, IDU8 >> KeyShift: {"u8", IDU8}, IDU16 >> KeyShift: {"u16", IDU16}, IDU32 >> KeyShift: {"u32", IDU32}, IDU64 >> KeyShift: {"u64", IDU64}, IDUsize >> KeyShift: {"usize", IDUsize}, IDBool >> KeyShift: {"bool", IDBool}, IDBuf1 >> KeyShift: {"buf1", IDBuf1}, IDBuf2 >> KeyShift: {"buf2", IDBuf2}, IDUnderscore >> KeyShift: {"_", IDUnderscore}, IDThis >> KeyShift: {"this", IDThis}, } var builtInsByName = map[string]ID{} func init() { for _, x := range builtInsByKey { if x.name != "" { builtInsByName[x.name] = x.id } } } // squiggles are built-in IDs that aren't alpha-numeric. var squiggles = [256]ID{ '(': IDOpenParen, ')': IDCloseParen, '[': IDOpenBracket, ']': IDCloseBracket, '{': IDOpenCurly, '}': IDCloseCurly, '.': IDDot, ',': IDComma, '?': IDQuestion, ':': IDColon, ';': IDSemicolon, // 1<': 1 << KeyShift, } type suffixLexer struct { suffix string id ID } // lexers lex ambiguous 1-byte squiggles. For example, "&" might be the start // of "&^" or "&=". // // The order of the []suffixLexer elements matters. The first match wins. Since // we want to lex greedily, longer suffixes should be earlier in the slice. var lexers = [256][]suffixLexer{ '!': { {"=", IDNotEq}, }, '&': { {"^=", IDAmpHatEq}, {"^", IDAmpHat}, {"=", IDAmpEq}, {"", IDAmp}, }, '|': { {"=", IDPipeEq}, {"", IDPipe}, }, '^': { {"=", IDHatEq}, {"", IDHat}, }, '+': { {"=", IDPlusEq}, {"", IDPlus}, }, '-': { {"=", IDMinusEq}, {"", IDMinus}, }, '*': { {"=", IDStarEq}, {"", IDStar}, }, '/': { {"=", IDSlashEq}, {"", IDSlash}, }, '=': { {"=", IDEqEq}, {"", IDEq}, }, '<': { {"<=", IDShiftLEq}, {"<", IDShiftL}, {"=", IDLessEq}, {"", IDLessThan}, }, '>': { {">=", IDShiftREq}, {">", IDShiftR}, {"=", IDGreaterEq}, {"", IDGreaterThan}, }, } var unaryForms = [256]ID{ IDPlus >> KeyShift: IDXUnaryPlus, IDMinus >> KeyShift: IDXUnaryMinus, IDNot >> KeyShift: IDXUnaryNot, } var binaryForms = [256]ID{ IDPlus >> KeyShift: IDXBinaryPlus, IDMinus >> KeyShift: IDXBinaryMinus, IDStar >> KeyShift: IDXBinaryStar, IDSlash >> KeyShift: IDXBinarySlash, IDShiftL >> KeyShift: IDXBinaryShiftL, IDShiftR >> KeyShift: IDXBinaryShiftR, IDAmp >> KeyShift: IDXBinaryAmp, IDAmpHat >> KeyShift: IDXBinaryAmpHat, IDPipe >> KeyShift: IDXBinaryPipe, IDHat >> KeyShift: IDXBinaryHat, IDNotEq >> KeyShift: IDXBinaryNotEq, IDLessThan >> KeyShift: IDXBinaryLessThan, IDLessEq >> KeyShift: IDXBinaryLessEq, IDEqEq >> KeyShift: IDXBinaryEqEq, IDGreaterEq >> KeyShift: IDXBinaryGreaterEq, IDGreaterThan >> KeyShift: IDXBinaryGreaterThan, IDAnd >> KeyShift: IDXBinaryAnd, IDOr >> KeyShift: IDXBinaryOr, IDAs >> KeyShift: IDXBinaryAs, } var associativeForms = [256]ID{ IDPlus >> KeyShift: IDXAssociativePlus, IDStar >> KeyShift: IDXAssociativeStar, IDAmp >> KeyShift: IDXAssociativeAmp, IDPipe >> KeyShift: IDXAssociativePipe, IDHat >> KeyShift: IDXAssociativeHat, IDAnd >> KeyShift: IDXAssociativeAnd, IDOr >> KeyShift: IDXAssociativeOr, }