diff --git a/lang/parse/parse.go b/lang/parse/parse.go index 26e648cc..ce752998 100644 --- a/lang/parse/parse.go +++ b/lang/parse/parse.go @@ -309,7 +309,7 @@ func (p *parser) parseIdent() (t.ID, error) { return 0, fmt.Errorf(`parse: expected identifier at %s:%d`, p.filename, p.line()) } x := p.src[0] - if !x.IsIdent() { + if !x.ID.IsIdent() { got := p.tm.ByToken(x) return 0, fmt.Errorf(`parse: expected identifier, got %q at %s:%d`, got, p.filename, p.line()) } @@ -978,7 +978,7 @@ func (p *parser) parseOperand() (*a.Expr, error) { case t.KeyExclam, t.KeyQuestion: flags |= a.FlagsImpure | a.FlagsCallImpure - if p.src[0].Key() == t.KeyQuestion { + if p.src[0].ID.Key() == t.KeyQuestion { flags |= a.FlagsSuspendible | a.FlagsCallSuspendible } p.src = p.src[1:] diff --git a/lang/render/render.go b/lang/render/render.go index d0adf963..0e81dcfc 100644 --- a/lang/render/render.go +++ b/lang/render/render.go @@ -108,15 +108,13 @@ func Render(w io.Writer, tm *t.Map, src []t.Token, comments []string) (err error prevID, prevIsTightRight := t.ID(0), false for _, tok := range lineTokens { const ( - flagsCIL = t.FlagsClose | t.FlagsIdent | t.FlagsLiteral - flagsCIS = t.FlagsClose | t.FlagsIdent | t.FlagsStrLiteral - flagsUB = t.FlagsUnaryOp | t.FlagsBinaryOp + flagsUB = t.FlagsUnaryOp | t.FlagsBinaryOp ) - if prevID != 0 && !prevIsTightRight && !tok.IsTightLeft() { + if prevID != 0 && !prevIsTightRight && !tok.ID.IsTightLeft() { // The "(" token's tight-left-ness is context dependent. For // "f(x)", the "(" is tight-left. For "a * (b + c)", it is not. - if tok.ID.Key() != t.KeyOpenParen || prevID.Flags()&flagsCIS == 0 { + if tok.ID.Key() != t.KeyOpenParen || !isCloseIdentStrLiteral(prevID) { buf = append(buf, ' ') } } @@ -141,7 +139,7 @@ func Render(w io.Writer, tm *t.Map, src []t.Token, comments []string) (err error if prevID != 0 && tok.ID.Flags()&flagsUB == flagsUB { // Token-based (not ast.Node-based) heuristic for whether the // operator looks unary instead of binary. - prevIsTightRight = prevID.Flags()&flagsCIL == 0 + prevIsTightRight = !isCloseIdentLiteral(prevID) } prevID = tok.ID @@ -210,3 +208,13 @@ func appendTabs(buf []byte, n int) []byte { } return buf } + +func isCloseIdentLiteral(x t.ID) bool { + const flags = t.FlagsIdent | t.FlagsLiteral + return x.IsClose() || x.Flags()&flags != 0 +} + +func isCloseIdentStrLiteral(x t.ID) bool { + const flags = t.FlagsIdent | t.FlagsStrLiteral + return x.IsClose() || x.Flags()&flags != 0 +} diff --git a/lang/token/list.go b/lang/token/list.go index 45ee20af..ad35f82f 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -29,22 +29,15 @@ const MaxIntBits = 64 type Flags uint32 const ( - FlagsOther = Flags(0x0001) - FlagsUnaryOp = Flags(0x0002) - FlagsBinaryOp = Flags(0x0004) - FlagsAssociativeOp = Flags(0x0008) - FlagsLiteral = Flags(0x0010) - FlagsNumLiteral = Flags(0x0020) - FlagsStrLiteral = Flags(0x0040) - FlagsIdent = Flags(0x0080) - FlagsOpen = Flags(0x0100) - FlagsClose = Flags(0x0200) - FlagsTightLeft = Flags(0x0400) - FlagsTightRight = Flags(0x0800) - FlagsAssign = Flags(0x1000) - FlagsImplicitSemicolon = Flags(0x2000) - FlagsNumType = Flags(0x4000) - flagsUnused = Flags(0x8000) + FlagsOther = Flags(0x0001) + FlagsUnaryOp = Flags(0x0002) + FlagsBinaryOp = Flags(0x0004) + FlagsAssociativeOp = Flags(0x0008) + FlagsLiteral = Flags(0x0010) + FlagsNumLiteral = Flags(0x0020) + FlagsStrLiteral = Flags(0x0040) + FlagsIdent = Flags(0x0080) + flagsUnused = Flags(0x8000) ) // Key is the high 16 bits of an ID. It is the map key for a Map. @@ -76,20 +69,27 @@ func (x ID) Flags() Flags { return Flags(x & FlagsMask) } func (x ID) IsBuiltIn() bool { return Key(x>>KeyShift) < nBuiltInKeys } -func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } -func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } -func (x ID) IsAssociativeOp() bool { return Flags(x)&FlagsAssociativeOp != 0 } -func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } -func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 } -func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 } -func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 } -func (x ID) IsOpen() bool { return Flags(x)&FlagsOpen != 0 } -func (x ID) IsClose() bool { return Flags(x)&FlagsClose != 0 } -func (x ID) IsTightLeft() bool { return Flags(x)&FlagsTightLeft != 0 } -func (x ID) IsTightRight() bool { return Flags(x)&FlagsTightRight != 0 } -func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 } -func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 } -func (x ID) IsNumType() bool { return Flags(x)&FlagsNumType != 0 } +func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } +func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } +func (x ID) IsAssociativeOp() bool { return Flags(x)&FlagsAssociativeOp != 0 } + +func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } +func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 } +func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 } +func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 } + +func (x ID) IsOpen() bool { return x.Key() < Key(len(isOpen)) && isOpen[x.Key()] } +func (x ID) IsClose() bool { return x.Key() < Key(len(isClose)) && isClose[x.Key()] } +func (x ID) IsTightLeft() bool { return x.Key() < Key(len(isTightLeft)) && isTightLeft[x.Key()] } +func (x ID) IsTightRight() bool { return x.Key() < Key(len(isTightRight)) && isTightRight[x.Key()] } + +func (x ID) IsAssign() bool { return minAssignKey <= x.Key() && x.Key() <= maxAssignKey } +func (x ID) IsNumType() bool { return minNumTypeKey <= x.Key() && x.Key() <= maxNumTypeKey } + +func (x ID) IsImplicitSemicolon() bool { + return x.IsLiteral() || x.IsIdent() || + (x.Key() < Key(len(isImplicitSemicolon)) && isImplicitSemicolon[x.Key()]) +} func (x ID) IsXUnaryOp() bool { return x.Key().isXOp() && x.IsUnaryOp() } func (x ID) IsXBinaryOp() bool { return x.Key().isXOp() && x.IsBinaryOp() } @@ -131,30 +131,6 @@ type Token struct { Line uint32 } -func (t Token) Key() Key { return Key(t.ID >> KeyShift) } -func (t Token) Flags() Flags { return Flags(t.ID & FlagsMask) } - -func (t Token) IsBuiltIn() bool { return Key(t.ID>>KeyShift) < nBuiltInKeys } - -func (t Token) IsUnaryOp() bool { return Flags(t.ID)&FlagsUnaryOp != 0 } -func (t Token) IsBinaryOp() bool { return Flags(t.ID)&FlagsBinaryOp != 0 } -func (t Token) IsAssociativeOp() bool { return Flags(t.ID)&FlagsAssociativeOp != 0 } -func (t Token) IsLiteral() bool { return Flags(t.ID)&FlagsLiteral != 0 } -func (t Token) IsNumLiteral() bool { return Flags(t.ID)&FlagsNumLiteral != 0 } -func (t Token) IsStrLiteral() bool { return Flags(t.ID)&FlagsStrLiteral != 0 } -func (t Token) IsIdent() bool { return Flags(t.ID)&FlagsIdent != 0 } -func (t Token) IsOpen() bool { return Flags(t.ID)&FlagsOpen != 0 } -func (t Token) IsClose() bool { return Flags(t.ID)&FlagsClose != 0 } -func (t Token) IsTightLeft() bool { return Flags(t.ID)&FlagsTightLeft != 0 } -func (t Token) IsTightRight() bool { return Flags(t.ID)&FlagsTightRight != 0 } -func (t Token) IsAssign() bool { return Flags(t.ID)&FlagsAssign != 0 } -func (t Token) IsImplicitSemicolon() bool { return Flags(t.ID)&FlagsImplicitSemicolon != 0 } -func (t Token) IsNumType() bool { return Flags(t.ID)&FlagsNumType != 0 } - -func (t Token) IsXUnaryOp() bool { return t.ID.Key().isXOp() && t.ID.IsUnaryOp() } -func (t Token) IsXBinaryOp() bool { return t.ID.Key().isXOp() && t.ID.IsBinaryOp() } -func (t Token) IsXAssociativeOp() bool { return t.ID.Key().isXOp() && t.ID.IsAssociativeOp() } - // nBuiltInKeys is the number of built-in Keys. The packing is: // - Zero is invalid. // - [0x01, 0x0F] are reserved for implementation details. @@ -370,37 +346,40 @@ const ( IDInvalid = ID(0) IDDoubleZ = ID(0x01< maxKey { return 0, errors.New("token: too many distinct tokens") } - flags := FlagsImplicitSemicolon + flags := Flags(0) if numeric(name[0]) { flags |= FlagsLiteral | FlagsNumLiteral } else if name[0] == '"' { @@ -132,7 +132,7 @@ loop: if c <= ' ' { if c == '\n' { - if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() { + if len(tokens) > 0 && tokens[len(tokens)-1].ID.IsImplicitSemicolon() { tokens = append(tokens, Token{IDSemicolon, line}) } if line == maxLine {