Remove token ID/Key distinction, part 10 of N

This commit is contained in:
Nigel Tao committed 2018-04-15 15:44:31 +10:00
1 parent 10b6eaebb6
commit b24a929ec7
1 file changed
+171 -187
+171 -187
View File
@@ -18,21 +18,6 @@ package token
// integer types.
const MaxIntBits = 64
// Flags are the low 16 bits of an ID. For example, they signify whether a
// token is an operator, an identifier, etc.
//
// The flags are not exclusive. For example, the "+" token is both a unary and
// binary operator. It is also associative: (a + b) + c equals a + (b + c).
//
// A valid token will have non-zero flags. If none of the other flags apply,
// the FlagsOther bit will be set.
type Flags uint32
const (
FlagsOther = Flags(0x0001)
flagsUnused = Flags(0x8000)
)
// Key is the high 16 bits of an ID. It is the map key for a Map.
type Key uint32
@@ -56,8 +41,7 @@ func (x ID) UnaryForm() ID { return unaryForms[0xFF&(x>>KeyShift)] }
func (x ID) BinaryForm() ID { return binaryForms[0xFF&(x>>KeyShift)] }
func (x ID) AssociativeForm() ID { return associativeForms[0xFF&(x>>KeyShift)] }
func (x ID) Key() Key { return Key(x >> KeyShift) }
func (x ID) Flags() Flags { return Flags(x & FlagsMask) }
func (x ID) Key() Key { return Key(x >> KeyShift) }
func (x ID) IsBuiltIn() bool { return Key(x>>KeyShift) < nBuiltInKeys }
@@ -374,43 +358,43 @@ const (
const (
IDInvalid = ID(0)
IDDoubleZ = ID(0x01<<KeyShift | FlagsOther)
IDDiamond = ID(0x02<<KeyShift | FlagsOther)
IDDoubleZ = ID(0x01 << KeyShift)
IDDiamond = ID(0x02 << KeyShift)
IDOpenParen = ID(0x10<<KeyShift | FlagsOther)
IDCloseParen = ID(0x11<<KeyShift | FlagsOther)
IDOpenBracket = ID(0x12<<KeyShift | FlagsOther)
IDCloseBracket = ID(0x13<<KeyShift | FlagsOther)
IDOpenCurly = ID(0x14<<KeyShift | FlagsOther)
IDCloseCurly = ID(0x15<<KeyShift | FlagsOther)
IDOpenParen = ID(0x10 << KeyShift)
IDCloseParen = ID(0x11 << KeyShift)
IDOpenBracket = ID(0x12 << KeyShift)
IDCloseBracket = ID(0x13 << KeyShift)
IDOpenCurly = ID(0x14 << KeyShift)
IDCloseCurly = ID(0x15 << KeyShift)
IDDot = ID(0x18<<KeyShift | FlagsOther)
IDDotDot = ID(0x19<<KeyShift | FlagsOther)
IDComma = ID(0x1A<<KeyShift | FlagsOther)
IDExclam = ID(0x1B<<KeyShift | FlagsOther)
IDQuestion = ID(0x1C<<KeyShift | FlagsOther)
IDColon = ID(0x1D<<KeyShift | FlagsOther)
IDSemicolon = ID(0x1E<<KeyShift | FlagsOther)
IDDollar = ID(0x1F<<KeyShift | FlagsOther)
IDDot = ID(0x18 << KeyShift)
IDDotDot = ID(0x19 << KeyShift)
IDComma = ID(0x1A << KeyShift)
IDExclam = ID(0x1B << KeyShift)
IDQuestion = ID(0x1C << KeyShift)
IDColon = ID(0x1D << KeyShift)
IDSemicolon = ID(0x1E << KeyShift)
IDDollar = ID(0x1F << KeyShift)
)
const (
minAssignKey = 0x20
maxAssignKey = 0x2F
IDEq = ID(0x20<<KeyShift | FlagsOther)
IDPlusEq = ID(0x21<<KeyShift | FlagsOther)
IDMinusEq = ID(0x22<<KeyShift | FlagsOther)
IDStarEq = ID(0x23<<KeyShift | FlagsOther)
IDSlashEq = ID(0x24<<KeyShift | FlagsOther)
IDShiftLEq = ID(0x25<<KeyShift | FlagsOther)
IDShiftREq = ID(0x26<<KeyShift | FlagsOther)
IDAmpEq = ID(0x27<<KeyShift | FlagsOther)
IDAmpHatEq = ID(0x28<<KeyShift | FlagsOther)
IDPipeEq = ID(0x29<<KeyShift | FlagsOther)
IDHatEq = ID(0x2A<<KeyShift | FlagsOther)
IDPercentEq = ID(0x2B<<KeyShift | FlagsOther)
IDTildePlusEq = ID(0x2C<<KeyShift | FlagsOther)
IDEq = ID(0x20 << KeyShift)
IDPlusEq = ID(0x21 << KeyShift)
IDMinusEq = ID(0x22 << KeyShift)
IDStarEq = ID(0x23 << KeyShift)
IDSlashEq = ID(0x24 << KeyShift)
IDShiftLEq = ID(0x25 << KeyShift)
IDShiftREq = ID(0x26 << KeyShift)
IDAmpEq = ID(0x27 << KeyShift)
IDAmpHatEq = ID(0x28 << KeyShift)
IDPipeEq = ID(0x29 << KeyShift)
IDHatEq = ID(0x2A << KeyShift)
IDPercentEq = ID(0x2B << KeyShift)
IDTildePlusEq = ID(0x2C << KeyShift)
)
const (
@@ -419,180 +403,180 @@ const (
maxXOpKey = 0x7F
maxOpKey = 0x7F
IDPlus = ID(0x31<<KeyShift | FlagsOther)
IDMinus = ID(0x32<<KeyShift | FlagsOther)
IDStar = ID(0x33<<KeyShift | FlagsOther)
IDSlash = ID(0x34<<KeyShift | FlagsOther)
IDShiftL = ID(0x35<<KeyShift | FlagsOther)
IDShiftR = ID(0x36<<KeyShift | FlagsOther)
IDAmp = ID(0x37<<KeyShift | FlagsOther)
IDAmpHat = ID(0x38<<KeyShift | FlagsOther)
IDPipe = ID(0x39<<KeyShift | FlagsOther)
IDHat = ID(0x3A<<KeyShift | FlagsOther)
IDPercent = ID(0x3B<<KeyShift | FlagsOther)
IDTildePlus = ID(0x3C<<KeyShift | FlagsOther)
IDPlus = ID(0x31 << KeyShift)
IDMinus = ID(0x32 << KeyShift)
IDStar = ID(0x33 << KeyShift)
IDSlash = ID(0x34 << KeyShift)
IDShiftL = ID(0x35 << KeyShift)
IDShiftR = ID(0x36 << KeyShift)
IDAmp = ID(0x37 << KeyShift)
IDAmpHat = ID(0x38 << KeyShift)
IDPipe = ID(0x39 << KeyShift)
IDHat = ID(0x3A << KeyShift)
IDPercent = ID(0x3B << KeyShift)
IDTildePlus = ID(0x3C << KeyShift)
IDNotEq = ID(0x40<<KeyShift | FlagsOther)
IDLessThan = ID(0x41<<KeyShift | FlagsOther)
IDLessEq = ID(0x42<<KeyShift | FlagsOther)
IDEqEq = ID(0x43<<KeyShift | FlagsOther)
IDGreaterEq = ID(0x44<<KeyShift | FlagsOther)
IDGreaterThan = ID(0x45<<KeyShift | FlagsOther)
IDNotEq = ID(0x40 << KeyShift)
IDLessThan = ID(0x41 << KeyShift)
IDLessEq = ID(0x42 << KeyShift)
IDEqEq = ID(0x43 << KeyShift)
IDGreaterEq = ID(0x44 << KeyShift)
IDGreaterThan = ID(0x45 << KeyShift)
// TODO: sort these by name, when the list has stabilized.
IDAnd = ID(0x48<<KeyShift | FlagsOther)
IDOr = ID(0x49<<KeyShift | FlagsOther)
IDNot = ID(0x4A<<KeyShift | FlagsOther)
IDAs = ID(0x4B<<KeyShift | FlagsOther)
IDRef = ID(0x4C<<KeyShift | FlagsOther)
IDDeref = ID(0x4D<<KeyShift | FlagsOther)
IDAnd = ID(0x48 << KeyShift)
IDOr = ID(0x49 << KeyShift)
IDNot = ID(0x4A << KeyShift)
IDAs = ID(0x4B << KeyShift)
IDRef = ID(0x4C << KeyShift)
IDDeref = ID(0x4D << KeyShift)
// The IDXFoo IDs are not returned by the tokenizer. They are used by the
// ast.Node ID-typed fields to disambiguate e.g. unary vs binary plus.
IDXUnaryPlus = ID(0x50<<KeyShift | FlagsOther)
IDXUnaryMinus = ID(0x51<<KeyShift | FlagsOther)
IDXUnaryNot = ID(0x52<<KeyShift | FlagsOther)
IDXUnaryRef = ID(0x53<<KeyShift | FlagsOther)
IDXUnaryDeref = ID(0x54<<KeyShift | FlagsOther)
IDXUnaryPlus = ID(0x50 << KeyShift)
IDXUnaryMinus = ID(0x51 << KeyShift)
IDXUnaryNot = ID(0x52 << KeyShift)
IDXUnaryRef = ID(0x53 << KeyShift)
IDXUnaryDeref = ID(0x54 << KeyShift)
IDXBinaryPlus = ID(0x58<<KeyShift | FlagsOther)
IDXBinaryMinus = ID(0x59<<KeyShift | FlagsOther)
IDXBinaryStar = ID(0x5A<<KeyShift | FlagsOther)
IDXBinarySlash = ID(0x5B<<KeyShift | FlagsOther)
IDXBinaryShiftL = ID(0x5C<<KeyShift | FlagsOther)
IDXBinaryShiftR = ID(0x5D<<KeyShift | FlagsOther)
IDXBinaryAmp = ID(0x5E<<KeyShift | FlagsOther)
IDXBinaryAmpHat = ID(0x5F<<KeyShift | FlagsOther)
IDXBinaryPipe = ID(0x60<<KeyShift | FlagsOther)
IDXBinaryHat = ID(0x61<<KeyShift | FlagsOther)
IDXBinaryPercent = ID(0x62<<KeyShift | FlagsOther)
IDXBinaryNotEq = ID(0x63<<KeyShift | FlagsOther)
IDXBinaryLessThan = ID(0x64<<KeyShift | FlagsOther)
IDXBinaryLessEq = ID(0x65<<KeyShift | FlagsOther)
IDXBinaryEqEq = ID(0x66<<KeyShift | FlagsOther)
IDXBinaryGreaterEq = ID(0x67<<KeyShift | FlagsOther)
IDXBinaryGreaterThan = ID(0x68<<KeyShift | FlagsOther)
IDXBinaryAnd = ID(0x69<<KeyShift | FlagsOther)
IDXBinaryOr = ID(0x6A<<KeyShift | FlagsOther)
IDXBinaryAs = ID(0x6B<<KeyShift | FlagsOther)
IDXBinaryTildePlus = ID(0x6C<<KeyShift | FlagsOther)
IDXBinaryPlus = ID(0x58 << KeyShift)
IDXBinaryMinus = ID(0x59 << KeyShift)
IDXBinaryStar = ID(0x5A << KeyShift)
IDXBinarySlash = ID(0x5B << KeyShift)
IDXBinaryShiftL = ID(0x5C << KeyShift)
IDXBinaryShiftR = ID(0x5D << KeyShift)
IDXBinaryAmp = ID(0x5E << KeyShift)
IDXBinaryAmpHat = ID(0x5F << KeyShift)
IDXBinaryPipe = ID(0x60 << KeyShift)
IDXBinaryHat = ID(0x61 << KeyShift)
IDXBinaryPercent = ID(0x62 << KeyShift)
IDXBinaryNotEq = ID(0x63 << KeyShift)
IDXBinaryLessThan = ID(0x64 << KeyShift)
IDXBinaryLessEq = ID(0x65 << KeyShift)
IDXBinaryEqEq = ID(0x66 << KeyShift)
IDXBinaryGreaterEq = ID(0x67 << KeyShift)
IDXBinaryGreaterThan = ID(0x68 << KeyShift)
IDXBinaryAnd = ID(0x69 << KeyShift)
IDXBinaryOr = ID(0x6A << KeyShift)
IDXBinaryAs = ID(0x6B << KeyShift)
IDXBinaryTildePlus = ID(0x6C << KeyShift)
IDXAssociativePlus = ID(0x70<<KeyShift | FlagsOther)
IDXAssociativeStar = ID(0x71<<KeyShift | FlagsOther)
IDXAssociativeAmp = ID(0x72<<KeyShift | FlagsOther)
IDXAssociativePipe = ID(0x73<<KeyShift | FlagsOther)
IDXAssociativeHat = ID(0x74<<KeyShift | FlagsOther)
IDXAssociativeAnd = ID(0x75<<KeyShift | FlagsOther)
IDXAssociativeOr = ID(0x76<<KeyShift | FlagsOther)
IDXAssociativePlus = ID(0x70 << KeyShift)
IDXAssociativeStar = ID(0x71 << KeyShift)
IDXAssociativeAmp = ID(0x72 << KeyShift)
IDXAssociativePipe = ID(0x73 << KeyShift)
IDXAssociativeHat = ID(0x74 << KeyShift)
IDXAssociativeAnd = ID(0x75 << KeyShift)
IDXAssociativeOr = ID(0x76 << KeyShift)
)
const (
// TODO: sort these by name, when the list has stabilized.
IDFunc = ID(0x80<<KeyShift | FlagsOther)
IDAssert = ID(0x81<<KeyShift | FlagsOther)
IDWhile = ID(0x82<<KeyShift | FlagsOther)
IDIf = ID(0x83<<KeyShift | FlagsOther)
IDElse = ID(0x84<<KeyShift | FlagsOther)
IDReturn = ID(0x85<<KeyShift | FlagsOther)
IDBreak = ID(0x86<<KeyShift | FlagsOther)
IDContinue = ID(0x87<<KeyShift | FlagsOther)
IDStruct = ID(0x88<<KeyShift | FlagsOther)
IDUse = ID(0x89<<KeyShift | FlagsOther)
IDVar = ID(0x8A<<KeyShift | FlagsOther)
IDPre = ID(0x8B<<KeyShift | FlagsOther)
IDInv = ID(0x8C<<KeyShift | FlagsOther)
IDPost = ID(0x8D<<KeyShift | FlagsOther)
IDVia = ID(0x8E<<KeyShift | FlagsOther)
IDPub = ID(0x8F<<KeyShift | FlagsOther)
IDPri = ID(0x90<<KeyShift | FlagsOther)
IDError = ID(0x91<<KeyShift | FlagsOther)
IDSuspension = ID(0x92<<KeyShift | FlagsOther)
IDPackageID = ID(0x93<<KeyShift | FlagsOther)
IDConst = ID(0x94<<KeyShift | FlagsOther)
IDTry = ID(0x95<<KeyShift | FlagsOther)
IDIterate = ID(0x96<<KeyShift | FlagsOther)
IDYield = ID(0x97<<KeyShift | FlagsOther)
IDFunc = ID(0x80 << KeyShift)
IDAssert = ID(0x81 << KeyShift)
IDWhile = ID(0x82 << KeyShift)
IDIf = ID(0x83 << KeyShift)
IDElse = ID(0x84 << KeyShift)
IDReturn = ID(0x85 << KeyShift)
IDBreak = ID(0x86 << KeyShift)
IDContinue = ID(0x87 << KeyShift)
IDStruct = ID(0x88 << KeyShift)
IDUse = ID(0x89 << KeyShift)
IDVar = ID(0x8A << KeyShift)
IDPre = ID(0x8B << KeyShift)
IDInv = ID(0x8C << KeyShift)
IDPost = ID(0x8D << KeyShift)
IDVia = ID(0x8E << KeyShift)
IDPub = ID(0x8F << KeyShift)
IDPri = ID(0x90 << KeyShift)
IDError = ID(0x91 << KeyShift)
IDSuspension = ID(0x92 << KeyShift)
IDPackageID = ID(0x93 << KeyShift)
IDConst = ID(0x94 << KeyShift)
IDTry = ID(0x95 << KeyShift)
IDIterate = ID(0x96 << KeyShift)
IDYield = ID(0x97 << KeyShift)
IDArray = ID(0xA0<<KeyShift | FlagsOther)
IDNptr = ID(0xA1<<KeyShift | FlagsOther)
IDPtr = ID(0xA2<<KeyShift | FlagsOther)
IDSlice = ID(0xA3<<KeyShift | FlagsOther)
IDArray = ID(0xA0 << KeyShift)
IDNptr = ID(0xA1 << KeyShift)
IDPtr = ID(0xA2 << KeyShift)
IDSlice = ID(0xA3 << KeyShift)
)
const (
minBuiltInLiteralKey = 0xA4
maxBuiltInLiteralKey = 0xA7
IDFalse = ID(0xA4<<KeyShift | FlagsOther)
IDTrue = ID(0xA5<<KeyShift | FlagsOther)
IDZero = ID(0xA6<<KeyShift | FlagsOther)
IDFalse = ID(0xA4 << KeyShift)
IDTrue = ID(0xA5 << KeyShift)
IDZero = ID(0xA6 << KeyShift)
)
const (
minBuiltInIdentKey = 0xA8
maxBuiltInIdentKey = 0xFF
IDUnderscore = ID(0xA8<<KeyShift | FlagsOther)
IDThis = ID(0xA9<<KeyShift | FlagsOther)
IDIn = ID(0xAA<<KeyShift | FlagsOther)
IDOut = ID(0xAB<<KeyShift | FlagsOther)
IDSLICE = ID(0xAC<<KeyShift | FlagsOther)
IDBase = ID(0xAD<<KeyShift | FlagsOther)
IDUnderscore = ID(0xA8 << KeyShift)
IDThis = ID(0xA9 << KeyShift)
IDIn = ID(0xAA << KeyShift)
IDOut = ID(0xAB << KeyShift)
IDSLICE = ID(0xAC << KeyShift)
IDBase = ID(0xAD << KeyShift)
minNumTypeKey = 0xB0
maxNumTypeKey = 0xB7
IDI8 = ID(0xB0<<KeyShift | FlagsOther)
IDI16 = ID(0xB1<<KeyShift | FlagsOther)
IDI32 = ID(0xB2<<KeyShift | FlagsOther)
IDI64 = ID(0xB3<<KeyShift | FlagsOther)
IDU8 = ID(0xB4<<KeyShift | FlagsOther)
IDU16 = ID(0xB5<<KeyShift | FlagsOther)
IDU32 = ID(0xB6<<KeyShift | FlagsOther)
IDU64 = ID(0xB7<<KeyShift | FlagsOther)
IDI8 = ID(0xB0 << KeyShift)
IDI16 = ID(0xB1 << KeyShift)
IDI32 = ID(0xB2 << KeyShift)
IDI64 = ID(0xB3 << KeyShift)
IDU8 = ID(0xB4 << KeyShift)
IDU16 = ID(0xB5 << KeyShift)
IDU32 = ID(0xB6 << KeyShift)
IDU64 = ID(0xB7 << KeyShift)
IDBool = ID(0xB8<<KeyShift | FlagsOther)
IDIOReader = ID(0xB9<<KeyShift | FlagsOther)
IDIOWriter = ID(0xBA<<KeyShift | FlagsOther)
IDStatus = ID(0xBB<<KeyShift | FlagsOther)
IDImageConfig = ID(0xBC<<KeyShift | FlagsOther)
IDBool = ID(0xB8 << KeyShift)
IDIOReader = ID(0xB9 << KeyShift)
IDIOWriter = ID(0xBA << KeyShift)
IDStatus = ID(0xBB << KeyShift)
IDImageConfig = ID(0xBC << KeyShift)
IDMark = ID(0xC0<<KeyShift | FlagsOther)
IDReadU8 = ID(0xC1<<KeyShift | FlagsOther)
IDReadU16BE = ID(0xC2<<KeyShift | FlagsOther)
IDReadU16LE = ID(0xC3<<KeyShift | FlagsOther)
IDReadU32BE = ID(0xC4<<KeyShift | FlagsOther)
IDReadU32LE = ID(0xC5<<KeyShift | FlagsOther)
IDReadU64BE = ID(0xC6<<KeyShift | FlagsOther)
IDReadU64LE = ID(0xC7<<KeyShift | FlagsOther)
IDSinceMark = ID(0xC8<<KeyShift | FlagsOther)
IDWriteU8 = ID(0xC9<<KeyShift | FlagsOther)
IDWriteU16BE = ID(0xCA<<KeyShift | FlagsOther)
IDWriteU16LE = ID(0xCB<<KeyShift | FlagsOther)
IDWriteU32BE = ID(0xCC<<KeyShift | FlagsOther)
IDWriteU32LE = ID(0xCD<<KeyShift | FlagsOther)
IDWriteU64BE = ID(0xCE<<KeyShift | FlagsOther)
IDWriteU64LE = ID(0xCF<<KeyShift | FlagsOther)
IDMark = ID(0xC0 << KeyShift)
IDReadU8 = ID(0xC1 << KeyShift)
IDReadU16BE = ID(0xC2 << KeyShift)
IDReadU16LE = ID(0xC3 << KeyShift)
IDReadU32BE = ID(0xC4 << KeyShift)
IDReadU32LE = ID(0xC5 << KeyShift)
IDReadU64BE = ID(0xC6 << KeyShift)
IDReadU64LE = ID(0xC7 << KeyShift)
IDSinceMark = ID(0xC8 << KeyShift)
IDWriteU8 = ID(0xC9 << KeyShift)
IDWriteU16BE = ID(0xCA << KeyShift)
IDWriteU16LE = ID(0xCB << KeyShift)
IDWriteU32BE = ID(0xCC << KeyShift)
IDWriteU32LE = ID(0xCD << KeyShift)
IDWriteU64BE = ID(0xCE << KeyShift)
IDWriteU64LE = ID(0xCF << KeyShift)
IDIsError = ID(0xD0<<KeyShift | FlagsOther)
IDIsOK = ID(0xD1<<KeyShift | FlagsOther)
IDIsSuspension = ID(0xD2<<KeyShift | FlagsOther)
IDCopyFromHistory32 = ID(0xD3<<KeyShift | FlagsOther)
IDCopyFromReader32 = ID(0xD4<<KeyShift | FlagsOther)
IDCopyFromSlice = ID(0xD5<<KeyShift | FlagsOther)
IDCopyFromSlice32 = ID(0xD6<<KeyShift | FlagsOther)
IDSkip32 = ID(0xD7<<KeyShift | FlagsOther)
IDSkip64 = ID(0xD8<<KeyShift | FlagsOther)
IDLength = ID(0xD9<<KeyShift | FlagsOther)
IDAvailable = ID(0xDA<<KeyShift | FlagsOther)
IDPrefix = ID(0xDB<<KeyShift | FlagsOther)
IDSuffix = ID(0xDC<<KeyShift | FlagsOther)
IDLimit = ID(0xDD<<KeyShift | FlagsOther)
IDLowBits = ID(0xDE<<KeyShift | FlagsOther)
IDHighBits = ID(0xDF<<KeyShift | FlagsOther)
IDUnreadU8 = ID(0xE0<<KeyShift | FlagsOther)
IDIsMarked = ID(0xE1<<KeyShift | FlagsOther)
IDIsError = ID(0xD0 << KeyShift)
IDIsOK = ID(0xD1 << KeyShift)
IDIsSuspension = ID(0xD2 << KeyShift)
IDCopyFromHistory32 = ID(0xD3 << KeyShift)
IDCopyFromReader32 = ID(0xD4 << KeyShift)
IDCopyFromSlice = ID(0xD5 << KeyShift)
IDCopyFromSlice32 = ID(0xD6 << KeyShift)
IDSkip32 = ID(0xD7 << KeyShift)
IDSkip64 = ID(0xD8 << KeyShift)
IDLength = ID(0xD9 << KeyShift)
IDAvailable = ID(0xDA << KeyShift)
IDPrefix = ID(0xDB << KeyShift)
IDSuffix = ID(0xDC << KeyShift)
IDLimit = ID(0xDD << KeyShift)
IDLowBits = ID(0xDE << KeyShift)
IDHighBits = ID(0xDF << KeyShift)
IDUnreadU8 = ID(0xE0 << KeyShift)
IDIsMarked = ID(0xE1 << KeyShift)
)
var builtInsByKey = [nBuiltInKeys]struct {