Remove token ID/Key distinction, part 1 of N

This commit is contained in:
Nigel Tao committed 2018-04-15 10:41:18 +10:00
1 parent 06ff3d3369
commit afd7429e9a
4 files changed
+190 -151

No files matched your search

+2 -2
View File
@@ -309,7 +309,7 @@ func (p *parser) parseIdent() (t.ID, error) {
return 0, fmt.Errorf(`parse: expected identifier at %s:%d`, p.filename, p.line())
}
x := p.src[0]
if !x.IsIdent() {
if !x.ID.IsIdent() {
got := p.tm.ByToken(x)
return 0, fmt.Errorf(`parse: expected identifier, got %q at %s:%d`, got, p.filename, p.line())
}
@@ -978,7 +978,7 @@ func (p *parser) parseOperand() (*a.Expr, error) {
case t.KeyExclam, t.KeyQuestion:
flags |= a.FlagsImpure | a.FlagsCallImpure
if p.src[0].Key() == t.KeyQuestion {
if p.src[0].ID.Key() == t.KeyQuestion {
flags |= a.FlagsSuspendible | a.FlagsCallSuspendible
}
p.src = p.src[1:]
+14 -6
View File
@@ -108,15 +108,13 @@ func Render(w io.Writer, tm *t.Map, src []t.Token, comments []string) (err error
prevID, prevIsTightRight := t.ID(0), false
for _, tok := range lineTokens {
const (
flagsCIL = t.FlagsClose | t.FlagsIdent | t.FlagsLiteral
flagsCIS = t.FlagsClose | t.FlagsIdent | t.FlagsStrLiteral
flagsUB = t.FlagsUnaryOp | t.FlagsBinaryOp
flagsUB = t.FlagsUnaryOp | t.FlagsBinaryOp
)
if prevID != 0 && !prevIsTightRight && !tok.IsTightLeft() {
if prevID != 0 && !prevIsTightRight && !tok.ID.IsTightLeft() {
// The "(" token's tight-left-ness is context dependent. For
// "f(x)", the "(" is tight-left. For "a * (b + c)", it is not.
if tok.ID.Key() != t.KeyOpenParen || prevID.Flags()&flagsCIS == 0 {
if tok.ID.Key() != t.KeyOpenParen || !isCloseIdentStrLiteral(prevID) {
buf = append(buf, ' ')
}
}
@@ -141,7 +139,7 @@ func Render(w io.Writer, tm *t.Map, src []t.Token, comments []string) (err error
if prevID != 0 && tok.ID.Flags()&flagsUB == flagsUB {
// Token-based (not ast.Node-based) heuristic for whether the
// operator looks unary instead of binary.
prevIsTightRight = prevID.Flags()&flagsCIL == 0
prevIsTightRight = !isCloseIdentLiteral(prevID)
}
prevID = tok.ID
@@ -210,3 +208,13 @@ func appendTabs(buf []byte, n int) []byte {
}
return buf
}
func isCloseIdentLiteral(x t.ID) bool {
const flags = t.FlagsIdent | t.FlagsLiteral
return x.IsClose() || x.Flags()&flags != 0
}
func isCloseIdentStrLiteral(x t.ID) bool {
const flags = t.FlagsIdent | t.FlagsStrLiteral
return x.IsClose() || x.Flags()&flags != 0
}
+172 -141
View File
@@ -29,22 +29,15 @@ const MaxIntBits = 64
type Flags uint32
const (
FlagsOther = Flags(0x0001)
FlagsUnaryOp = Flags(0x0002)
FlagsBinaryOp = Flags(0x0004)
FlagsAssociativeOp = Flags(0x0008)
FlagsLiteral = Flags(0x0010)
FlagsNumLiteral = Flags(0x0020)
FlagsStrLiteral = Flags(0x0040)
FlagsIdent = Flags(0x0080)
FlagsOpen = Flags(0x0100)
FlagsClose = Flags(0x0200)
FlagsTightLeft = Flags(0x0400)
FlagsTightRight = Flags(0x0800)
FlagsAssign = Flags(0x1000)
FlagsImplicitSemicolon = Flags(0x2000)
FlagsNumType = Flags(0x4000)
flagsUnused = Flags(0x8000)
FlagsOther = Flags(0x0001)
FlagsUnaryOp = Flags(0x0002)
FlagsBinaryOp = Flags(0x0004)
FlagsAssociativeOp = Flags(0x0008)
FlagsLiteral = Flags(0x0010)
FlagsNumLiteral = Flags(0x0020)
FlagsStrLiteral = Flags(0x0040)
FlagsIdent = Flags(0x0080)
flagsUnused = Flags(0x8000)
)
// Key is the high 16 bits of an ID. It is the map key for a Map.
@@ -76,20 +69,27 @@ func (x ID) Flags() Flags { return Flags(x & FlagsMask) }
func (x ID) IsBuiltIn() bool { return Key(x>>KeyShift) < nBuiltInKeys }
func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 }
func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 }
func (x ID) IsAssociativeOp() bool { return Flags(x)&FlagsAssociativeOp != 0 }
func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 }
func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 }
func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 }
func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 }
func (x ID) IsOpen() bool { return Flags(x)&FlagsOpen != 0 }
func (x ID) IsClose() bool { return Flags(x)&FlagsClose != 0 }
func (x ID) IsTightLeft() bool { return Flags(x)&FlagsTightLeft != 0 }
func (x ID) IsTightRight() bool { return Flags(x)&FlagsTightRight != 0 }
func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 }
func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 }
func (x ID) IsNumType() bool { return Flags(x)&FlagsNumType != 0 }
func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 }
func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 }
func (x ID) IsAssociativeOp() bool { return Flags(x)&FlagsAssociativeOp != 0 }
func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 }
func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 }
func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 }
func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 }
func (x ID) IsOpen() bool { return x.Key() < Key(len(isOpen)) && isOpen[x.Key()] }
func (x ID) IsClose() bool { return x.Key() < Key(len(isClose)) && isClose[x.Key()] }
func (x ID) IsTightLeft() bool { return x.Key() < Key(len(isTightLeft)) && isTightLeft[x.Key()] }
func (x ID) IsTightRight() bool { return x.Key() < Key(len(isTightRight)) && isTightRight[x.Key()] }
func (x ID) IsAssign() bool { return minAssignKey <= x.Key() && x.Key() <= maxAssignKey }
func (x ID) IsNumType() bool { return minNumTypeKey <= x.Key() && x.Key() <= maxNumTypeKey }
func (x ID) IsImplicitSemicolon() bool {
return x.IsLiteral() || x.IsIdent() ||
(x.Key() < Key(len(isImplicitSemicolon)) && isImplicitSemicolon[x.Key()])
}
func (x ID) IsXUnaryOp() bool { return x.Key().isXOp() && x.IsUnaryOp() }
func (x ID) IsXBinaryOp() bool { return x.Key().isXOp() && x.IsBinaryOp() }
@@ -131,30 +131,6 @@ type Token struct {
Line uint32
}
func (t Token) Key() Key { return Key(t.ID >> KeyShift) }
func (t Token) Flags() Flags { return Flags(t.ID & FlagsMask) }
func (t Token) IsBuiltIn() bool { return Key(t.ID>>KeyShift) < nBuiltInKeys }
func (t Token) IsUnaryOp() bool { return Flags(t.ID)&FlagsUnaryOp != 0 }
func (t Token) IsBinaryOp() bool { return Flags(t.ID)&FlagsBinaryOp != 0 }
func (t Token) IsAssociativeOp() bool { return Flags(t.ID)&FlagsAssociativeOp != 0 }
func (t Token) IsLiteral() bool { return Flags(t.ID)&FlagsLiteral != 0 }
func (t Token) IsNumLiteral() bool { return Flags(t.ID)&FlagsNumLiteral != 0 }
func (t Token) IsStrLiteral() bool { return Flags(t.ID)&FlagsStrLiteral != 0 }
func (t Token) IsIdent() bool { return Flags(t.ID)&FlagsIdent != 0 }
func (t Token) IsOpen() bool { return Flags(t.ID)&FlagsOpen != 0 }
func (t Token) IsClose() bool { return Flags(t.ID)&FlagsClose != 0 }
func (t Token) IsTightLeft() bool { return Flags(t.ID)&FlagsTightLeft != 0 }
func (t Token) IsTightRight() bool { return Flags(t.ID)&FlagsTightRight != 0 }
func (t Token) IsAssign() bool { return Flags(t.ID)&FlagsAssign != 0 }
func (t Token) IsImplicitSemicolon() bool { return Flags(t.ID)&FlagsImplicitSemicolon != 0 }
func (t Token) IsNumType() bool { return Flags(t.ID)&FlagsNumType != 0 }
func (t Token) IsXUnaryOp() bool { return t.ID.Key().isXOp() && t.ID.IsUnaryOp() }
func (t Token) IsXBinaryOp() bool { return t.ID.Key().isXOp() && t.ID.IsBinaryOp() }
func (t Token) IsXAssociativeOp() bool { return t.ID.Key().isXOp() && t.ID.IsAssociativeOp() }
// nBuiltInKeys is the number of built-in Keys. The packing is:
// - Zero is invalid.
// - [0x01, 0x0F] are reserved for implementation details.
@@ -370,37 +346,40 @@ const (
IDInvalid = ID(0)
IDDoubleZ = ID(0x01<<KeyShift | FlagsOther)
IDDiamond = ID(0x02<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDDiamond = ID(0x02<<KeyShift | FlagsIdent)
IDOpenParen = ID(0x10<<KeyShift | FlagsOpen | FlagsTightRight)
IDCloseParen = ID(0x11<<KeyShift | FlagsClose | FlagsTightLeft | FlagsImplicitSemicolon)
IDOpenBracket = ID(0x12<<KeyShift | FlagsOpen | FlagsTightLeft | FlagsTightRight)
IDCloseBracket = ID(0x13<<KeyShift | FlagsClose | FlagsTightLeft | FlagsImplicitSemicolon)
IDOpenCurly = ID(0x14<<KeyShift | FlagsOpen)
IDCloseCurly = ID(0x15<<KeyShift | FlagsClose | FlagsImplicitSemicolon)
IDOpenParen = ID(0x10<<KeyShift | FlagsOther)
IDCloseParen = ID(0x11<<KeyShift | FlagsOther)
IDOpenBracket = ID(0x12<<KeyShift | FlagsOther)
IDCloseBracket = ID(0x13<<KeyShift | FlagsOther)
IDOpenCurly = ID(0x14<<KeyShift | FlagsOther)
IDCloseCurly = ID(0x15<<KeyShift | FlagsOther)
IDDot = ID(0x18<<KeyShift | FlagsTightLeft | FlagsTightRight)
IDDotDot = ID(0x19<<KeyShift | FlagsTightLeft | FlagsTightRight)
IDComma = ID(0x1A<<KeyShift | FlagsTightLeft)
IDExclam = ID(0x1B<<KeyShift | FlagsTightLeft | FlagsTightRight)
IDQuestion = ID(0x1C<<KeyShift | FlagsTightLeft | FlagsTightRight)
IDColon = ID(0x1D<<KeyShift | FlagsTightLeft | FlagsTightRight)
IDSemicolon = ID(0x1E<<KeyShift | FlagsTightLeft)
IDDollar = ID(0x1F<<KeyShift | FlagsTightRight)
IDDot = ID(0x18<<KeyShift | FlagsOther)
IDDotDot = ID(0x19<<KeyShift | FlagsOther)
IDComma = ID(0x1A<<KeyShift | FlagsOther)
IDExclam = ID(0x1B<<KeyShift | FlagsOther)
IDQuestion = ID(0x1C<<KeyShift | FlagsOther)
IDColon = ID(0x1D<<KeyShift | FlagsOther)
IDSemicolon = ID(0x1E<<KeyShift | FlagsOther)
IDDollar = ID(0x1F<<KeyShift | FlagsOther)
IDEq = ID(0x20<<KeyShift | FlagsAssign)
IDPlusEq = ID(0x21<<KeyShift | FlagsAssign)
IDMinusEq = ID(0x22<<KeyShift | FlagsAssign)
IDStarEq = ID(0x23<<KeyShift | FlagsAssign)
IDSlashEq = ID(0x24<<KeyShift | FlagsAssign)
IDShiftLEq = ID(0x25<<KeyShift | FlagsAssign)
IDShiftREq = ID(0x26<<KeyShift | FlagsAssign)
IDAmpEq = ID(0x27<<KeyShift | FlagsAssign)
IDAmpHatEq = ID(0x28<<KeyShift | FlagsAssign)
IDPipeEq = ID(0x29<<KeyShift | FlagsAssign)
IDHatEq = ID(0x2A<<KeyShift | FlagsAssign)
IDPercentEq = ID(0x2B<<KeyShift | FlagsAssign)
IDTildePlusEq = ID(0x2C<<KeyShift | FlagsAssign)
minAssignKey = 0x20
maxAssignKey = 0x2F
IDEq = ID(0x20<<KeyShift | FlagsOther)
IDPlusEq = ID(0x21<<KeyShift | FlagsOther)
IDMinusEq = ID(0x22<<KeyShift | FlagsOther)
IDStarEq = ID(0x23<<KeyShift | FlagsOther)
IDSlashEq = ID(0x24<<KeyShift | FlagsOther)
IDShiftLEq = ID(0x25<<KeyShift | FlagsOther)
IDShiftREq = ID(0x26<<KeyShift | FlagsOther)
IDAmpEq = ID(0x27<<KeyShift | FlagsOther)
IDAmpHatEq = ID(0x28<<KeyShift | FlagsOther)
IDPipeEq = ID(0x29<<KeyShift | FlagsOther)
IDHatEq = ID(0x2A<<KeyShift | FlagsOther)
IDPercentEq = ID(0x2B<<KeyShift | FlagsOther)
IDTildePlusEq = ID(0x2C<<KeyShift | FlagsOther)
IDPlus = ID(0x31<<KeyShift | FlagsBinaryOp | FlagsUnaryOp | FlagsAssociativeOp)
IDMinus = ID(0x32<<KeyShift | FlagsBinaryOp | FlagsUnaryOp)
@@ -436,9 +415,9 @@ const (
IDWhile = ID(0x52<<KeyShift | FlagsOther)
IDIf = ID(0x53<<KeyShift | FlagsOther)
IDElse = ID(0x54<<KeyShift | FlagsOther)
IDReturn = ID(0x55<<KeyShift | FlagsOther | FlagsImplicitSemicolon)
IDBreak = ID(0x56<<KeyShift | FlagsOther | FlagsImplicitSemicolon)
IDContinue = ID(0x57<<KeyShift | FlagsOther | FlagsImplicitSemicolon)
IDReturn = ID(0x55<<KeyShift | FlagsOther)
IDBreak = ID(0x56<<KeyShift | FlagsOther)
IDContinue = ID(0x57<<KeyShift | FlagsOther)
IDStruct = ID(0x58<<KeyShift | FlagsOther)
IDUse = ID(0x59<<KeyShift | FlagsOther)
IDVar = ID(0x5A<<KeyShift | FlagsOther)
@@ -461,66 +440,70 @@ const (
IDPtr = ID(0x72<<KeyShift | FlagsOther)
IDSlice = ID(0x73<<KeyShift | FlagsOther)
IDFalse = ID(0x74<<KeyShift | FlagsLiteral | FlagsImplicitSemicolon)
IDTrue = ID(0x75<<KeyShift | FlagsLiteral | FlagsImplicitSemicolon)
IDZero = ID(0x76<<KeyShift | FlagsLiteral | FlagsImplicitSemicolon | FlagsNumLiteral)
IDFalse = ID(0x74<<KeyShift | FlagsLiteral)
IDTrue = ID(0x75<<KeyShift | FlagsLiteral)
IDZero = ID(0x76<<KeyShift | FlagsLiteral | FlagsNumLiteral)
IDUnderscore = ID(0x78<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDThis = ID(0x79<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDIn = ID(0x7A<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDOut = ID(0x7B<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDSLICE = ID(0x7C<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDBase = ID(0x7D<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDUnderscore = ID(0x78<<KeyShift | FlagsIdent)
IDThis = ID(0x79<<KeyShift | FlagsIdent)
IDIn = ID(0x7A<<KeyShift | FlagsIdent)
IDOut = ID(0x7B<<KeyShift | FlagsIdent)
IDSLICE = ID(0x7C<<KeyShift | FlagsIdent)
IDBase = ID(0x7D<<KeyShift | FlagsIdent)
IDI8 = ID(0x80<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDI16 = ID(0x81<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDI32 = ID(0x82<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDI64 = ID(0x83<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDU8 = ID(0x84<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDU16 = ID(0x85<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDU32 = ID(0x86<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDU64 = ID(0x87<<KeyShift | FlagsIdent | FlagsImplicitSemicolon | FlagsNumType)
IDBool = ID(0x88<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDIOReader = ID(0x89<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDIOWriter = ID(0x8A<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDStatus = ID(0x8B<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDImageConfig = ID(0x8C<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
minNumTypeKey = 0x80
maxNumTypeKey = 0x87
IDMark = ID(0x90<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU8 = ID(0x91<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU16BE = ID(0x92<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU16LE = ID(0x93<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU32BE = ID(0x94<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU32LE = ID(0x95<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU64BE = ID(0x96<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDReadU64LE = ID(0x97<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDSinceMark = ID(0x98<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU8 = ID(0x99<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU16BE = ID(0x9A<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU16LE = ID(0x9B<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU32BE = ID(0x9C<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU32LE = ID(0x9D<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU64BE = ID(0x9E<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDWriteU64LE = ID(0x9F<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDI8 = ID(0x80<<KeyShift | FlagsIdent)
IDI16 = ID(0x81<<KeyShift | FlagsIdent)
IDI32 = ID(0x82<<KeyShift | FlagsIdent)
IDI64 = ID(0x83<<KeyShift | FlagsIdent)
IDU8 = ID(0x84<<KeyShift | FlagsIdent)
IDU16 = ID(0x85<<KeyShift | FlagsIdent)
IDU32 = ID(0x86<<KeyShift | FlagsIdent)
IDU64 = ID(0x87<<KeyShift | FlagsIdent)
IDIsError = ID(0xA0<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDIsOK = ID(0xA1<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDIsSuspension = ID(0xA2<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDCopyFromHistory32 = ID(0xA3<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDCopyFromReader32 = ID(0xA4<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDCopyFromSlice = ID(0xA5<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDCopyFromSlice32 = ID(0xA6<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDSkip32 = ID(0xA7<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDSkip64 = ID(0xA8<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDLength = ID(0xA9<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDAvailable = ID(0xAA<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDPrefix = ID(0xAB<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDSuffix = ID(0xAC<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDLimit = ID(0xAD<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDLowBits = ID(0xAE<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDHighBits = ID(0xAF<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDUnreadU8 = ID(0xB0<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDIsMarked = ID(0xB1<<KeyShift | FlagsIdent | FlagsImplicitSemicolon)
IDBool = ID(0x88<<KeyShift | FlagsIdent)
IDIOReader = ID(0x89<<KeyShift | FlagsIdent)
IDIOWriter = ID(0x8A<<KeyShift | FlagsIdent)
IDStatus = ID(0x8B<<KeyShift | FlagsIdent)
IDImageConfig = ID(0x8C<<KeyShift | FlagsIdent)
IDMark = ID(0x90<<KeyShift | FlagsIdent)
IDReadU8 = ID(0x91<<KeyShift | FlagsIdent)
IDReadU16BE = ID(0x92<<KeyShift | FlagsIdent)
IDReadU16LE = ID(0x93<<KeyShift | FlagsIdent)
IDReadU32BE = ID(0x94<<KeyShift | FlagsIdent)
IDReadU32LE = ID(0x95<<KeyShift | FlagsIdent)
IDReadU64BE = ID(0x96<<KeyShift | FlagsIdent)
IDReadU64LE = ID(0x97<<KeyShift | FlagsIdent)
IDSinceMark = ID(0x98<<KeyShift | FlagsIdent)
IDWriteU8 = ID(0x99<<KeyShift | FlagsIdent)
IDWriteU16BE = ID(0x9A<<KeyShift | FlagsIdent)
IDWriteU16LE = ID(0x9B<<KeyShift | FlagsIdent)
IDWriteU32BE = ID(0x9C<<KeyShift | FlagsIdent)
IDWriteU32LE = ID(0x9D<<KeyShift | FlagsIdent)
IDWriteU64BE = ID(0x9E<<KeyShift | FlagsIdent)
IDWriteU64LE = ID(0x9F<<KeyShift | FlagsIdent)
IDIsError = ID(0xA0<<KeyShift | FlagsIdent)
IDIsOK = ID(0xA1<<KeyShift | FlagsIdent)
IDIsSuspension = ID(0xA2<<KeyShift | FlagsIdent)
IDCopyFromHistory32 = ID(0xA3<<KeyShift | FlagsIdent)
IDCopyFromReader32 = ID(0xA4<<KeyShift | FlagsIdent)
IDCopyFromSlice = ID(0xA5<<KeyShift | FlagsIdent)
IDCopyFromSlice32 = ID(0xA6<<KeyShift | FlagsIdent)
IDSkip32 = ID(0xA7<<KeyShift | FlagsIdent)
IDSkip64 = ID(0xA8<<KeyShift | FlagsIdent)
IDLength = ID(0xA9<<KeyShift | FlagsIdent)
IDAvailable = ID(0xAA<<KeyShift | FlagsIdent)
IDPrefix = ID(0xAB<<KeyShift | FlagsIdent)
IDSuffix = ID(0xAC<<KeyShift | FlagsIdent)
IDLimit = ID(0xAD<<KeyShift | FlagsIdent)
IDLowBits = ID(0xAE<<KeyShift | FlagsIdent)
IDHighBits = ID(0xAF<<KeyShift | FlagsIdent)
IDUnreadU8 = ID(0xB0<<KeyShift | FlagsIdent)
IDIsMarked = ID(0xB1<<KeyShift | FlagsIdent)
)
// The IDXFoo IDs are not returned by the tokenizer. They are used by the
@@ -930,3 +913,51 @@ var associativeForms = [256]ID{
KeyAnd: IDXAssociativeAnd,
KeyOr: IDXAssociativeOr,
}
var isOpen = [...]bool{
KeyOpenParen: true,
KeyOpenBracket: true,
KeyOpenCurly: true,
}
var isClose = [...]bool{
KeyCloseParen: true,
KeyCloseBracket: true,
KeyCloseCurly: true,
}
var isTightLeft = [...]bool{
KeyCloseParen: true,
KeyOpenBracket: true,
KeyCloseBracket: true,
KeyDot: true,
KeyDotDot: true,
KeyComma: true,
KeyExclam: true,
KeyQuestion: true,
KeyColon: true,
KeySemicolon: true,
}
var isTightRight = [...]bool{
KeyOpenParen: true,
KeyOpenBracket: true,
KeyDot: true,
KeyDotDot: true,
KeyExclam: true,
KeyQuestion: true,
KeyColon: true,
KeyDollar: true,
}
var isImplicitSemicolon = [...]bool{
KeyCloseParen: true,
KeyCloseBracket: true,
KeyCloseCurly: true,
KeyReturn: true,
KeyBreak: true,
KeyContinue: true,
}
+2 -2
View File
@@ -54,7 +54,7 @@ func (m *Map) Insert(name string) (ID, error) {
if key > maxKey {
return 0, errors.New("token: too many distinct tokens")
}
flags := FlagsImplicitSemicolon
flags := Flags(0)
if numeric(name[0]) {
flags |= FlagsLiteral | FlagsNumLiteral
} else if name[0] == '"' {
@@ -132,7 +132,7 @@ loop:
if c <= ' ' {
if c == '\n' {
if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() {
if len(tokens) > 0 && tokens[len(tokens)-1].ID.IsImplicitSemicolon() {
tokens = append(tokens, Token{IDSemicolon, line})
}
if line == maxLine {