From 10b6eaebb6ff72b905e3ebcb502d18e016dd5dbc Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 15 Apr 2018 15:41:48 +1000 Subject: [PATCH] Remove token ID/Key distinction, part 9 of N --- lang/render/render.go | 14 ++-- lang/token/list.go | 172 ++++++++++++++++++++++++++---------------- lang/token/token.go | 10 +-- 3 files changed, 112 insertions(+), 84 deletions(-) diff --git a/lang/render/render.go b/lang/render/render.go index 52ac2edd..6e901e51 100644 --- a/lang/render/render.go +++ b/lang/render/render.go @@ -110,7 +110,7 @@ func Render(w io.Writer, tm *t.Map, src []t.Token, comments []string) (err error if prevID != 0 && !prevIsTightRight && !tok.ID.IsTightLeft() { // The "(" token's tight-left-ness is context dependent. For // "f(x)", the "(" is tight-left. For "a * (b + c)", it is not. - if tok.ID.Key() != t.KeyOpenParen || !isCloseIdentStrLiteral(prevID) { + if tok.ID.Key() != t.KeyOpenParen || !isCloseIdentStrLiteral(tm, prevID) { buf = append(buf, ' ') } } @@ -135,7 +135,7 @@ func Render(w io.Writer, tm *t.Map, src []t.Token, comments []string) (err error if prevID != 0 && tok.ID.IsUnaryOp() && tok.ID.IsBinaryOp() { // Token-based (not ast.Node-based) heuristic for whether the // operator looks unary instead of binary. - prevIsTightRight = !isCloseIdentLiteral(prevID) + prevIsTightRight = !isCloseIdentLiteral(tm, prevID) } prevID = tok.ID @@ -205,12 +205,10 @@ func appendTabs(buf []byte, n int) []byte { return buf } -func isCloseIdentLiteral(x t.ID) bool { - const flags = t.FlagsIdent | t.FlagsLiteral - return x.IsClose() || x.Flags()&flags != 0 +func isCloseIdentLiteral(tm *t.Map, x t.ID) bool { + return x.IsClose() || x.IsIdent(tm) || x.IsLiteral(tm) } -func isCloseIdentStrLiteral(x t.ID) bool { - const flags = t.FlagsIdent | t.FlagsStrLiteral - return x.IsClose() || x.Flags()&flags != 0 +func isCloseIdentStrLiteral(tm *t.Map, x t.ID) bool { + return x.IsClose() || x.IsIdent(tm) || x.IsStrLiteral(tm) } diff --git a/lang/token/list.go b/lang/token/list.go index c37447c3..a7a0ded1 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -29,12 +29,8 @@ const MaxIntBits = 64 type Flags uint32 const ( - FlagsOther = Flags(0x0001) - FlagsLiteral = Flags(0x0010) - FlagsNumLiteral = Flags(0x0020) - FlagsStrLiteral = Flags(0x0040) - FlagsIdent = Flags(0x0080) - flagsUnused = Flags(0x8000) + FlagsOther = Flags(0x0001) + flagsUnused = Flags(0x8000) ) // Key is the high 16 bits of an ID. It is the map key for a Map. @@ -73,10 +69,42 @@ func (x ID) IsAssociativeOp() bool { return minOpKey <= x.Key() && x.Key() <= maxOpKey && x.AssociativeForm() != 0 } -func (x ID) IsLiteral(m *Map) bool { return Flags(x)&FlagsLiteral != 0 } -func (x ID) IsNumLiteral(m *Map) bool { return Flags(x)&FlagsNumLiteral != 0 } -func (x ID) IsStrLiteral(m *Map) bool { return Flags(x)&FlagsStrLiteral != 0 } -func (x ID) IsIdent(m *Map) bool { return Flags(x)&FlagsIdent != 0 } +func (x ID) IsLiteral(m *Map) bool { + if k := x.Key(); k < nBuiltInKeys { + return minBuiltInLiteralKey <= k && k <= maxBuiltInLiteralKey + } else if s := m.ByKey(k); s != "" { + return !alpha(s[0]) + } + return false +} + +func (x ID) IsNumLiteral(m *Map) bool { + if k := x.Key(); k < nBuiltInKeys { + return k == KeyZero + } else if s := m.ByKey(k); s != "" { + return numeric(s[0]) + } + return false +} + +func (x ID) IsStrLiteral(m *Map) bool { + if k := x.Key(); k < nBuiltInKeys { + return false + } else if s := m.ByKey(k); s != "" { + return s[0] == '"' + } + return false +} + +func (x ID) IsIdent(m *Map) bool { + if k := x.Key(); k < nBuiltInKeys { + // TODO: move Diamond into the built-in ident space. + return k == KeyDiamond || (minBuiltInIdentKey <= k && k <= maxBuiltInIdentKey) + } else if s := m.ByKey(k); s != "" { + return alpha(s[0]) + } + return false +} func (x ID) IsOpen() bool { return x.Key() < Key(len(isOpen)) && isOpen[x.Key()] } func (x ID) IsClose() bool { return x.Key() < Key(len(isClose)) && isClose[x.Key()] } @@ -347,7 +375,7 @@ const ( IDInvalid = ID(0) IDDoubleZ = ID(0x01< maxKey { return 0, errors.New("token: too many distinct tokens") } - flags := Flags(0) - if numeric(name[0]) { - flags |= FlagsLiteral | FlagsNumLiteral - } else if name[0] == '"' { - flags |= FlagsLiteral | FlagsStrLiteral - } else { - flags |= FlagsIdent - } - id := ID(key<