From 9fe975213bd5710a90b2f4db317a3b318ae2e0ce Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 5 May 2017 16:15:11 +1000 Subject: [PATCH] Disambiguate unary vs binary plus' token.IDs. --- lang/ast/ast.go | 12 ++- lang/parse/parse.go | 20 +++- lang/token/list.go | 217 ++++++++++++++++++++++++++++++-------------- 3 files changed, 172 insertions(+), 77 deletions(-) diff --git a/lang/ast/ast.go b/lang/ast/ast.go index 87caa43b..dfd5b1fa 100644 --- a/lang/ast/ast.go +++ b/lang/ast/ast.go @@ -16,7 +16,7 @@ const ( KInvalid = Kind(iota) // KExpr is an expression, such as "i", "+j" or "k + l[m(n, o)].p": - // - ID0: <0|operator|IDOpenParen|IDOpenBracket|IDDot> + // - ID0: <0|operator|IDOpenParen|IDOpenBracket|IDColon|IDDot> // - ID1: <0|identifier name|literal> // - LHS: // - MHS: @@ -29,11 +29,13 @@ const ( // For unary operators, ID0 is the operator and RHS is the operand. // // For binary operators, ID0 is the operator and LHS and RHS are the - // operands. The LHS may be null for the "-" and "+" operators, which are - // also unary operators. + // operands. // - // For associative binary operators, both LHS and RHS may be null, and - // multiple operands are in List0. + // For associative operators, ID0 is the operator and List0 holds the + // operands. + // + // The ID0 operator is in disambiguous form. For example, IDUnaryPlus, + // IDBinaryPlus or IDAssociativePlus, not a bare IDPlus. // // For function calls, like "LHS(List0)", ID0 is IDOpenParen. // diff --git a/lang/parse/parse.go b/lang/parse/parse.go index ff252802..ade9b0e5 100644 --- a/lang/parse/parse.go +++ b/lang/parse/parse.go @@ -505,10 +505,14 @@ func (p *parser) parseExpr() (*a.Node, error) { return nil, err } - if !x.IsAssociative() || x != p.peekID() { + if !x.IsAssociativeOp() || x != p.peekID() { + id0 := x.BinaryForm() + if id0 == 0 { + return nil, fmt.Errorf("parse: internal error: no binary form for token.Key 0x%#02x", x.Key()) + } return &a.Node{ Kind: a.KExpr, - ID0: x, + ID0: id0, LHS: lhs, RHS: rhs, }, nil @@ -523,9 +527,13 @@ func (p *parser) parseExpr() (*a.Node, error) { } list0 = append(list0, arg) } + id0 := x.AssociativeForm() + if id0 == 0 { + return nil, fmt.Errorf("parse: internal error: no associative form for token.Key 0x%#02x", x.Key()) + } return &a.Node{ Kind: a.KExpr, - ID0: x, + ID0: id0, List0: list0, }, nil } @@ -553,9 +561,13 @@ func (p *parser) parseOperand() (*a.Node, error) { if err != nil { return nil, err } + id0 := x.UnaryForm() + if id0 == 0 { + return nil, fmt.Errorf("parse: internal error: no unary form for token.Key 0x%#02x", x.Key()) + } return &a.Node{ Kind: a.KExpr, - ID0: x, + ID0: id0, RHS: rhs, }, nil diff --git a/lang/token/list.go b/lang/token/list.go index ae60f936..fa29764b 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -17,7 +17,7 @@ const ( FlagsOther = Flags(0x0001) FlagsUnaryOp = Flags(0x0002) FlagsBinaryOp = Flags(0x0004) - FlagsAssociative = Flags(0x0008) + FlagsAssociativeOp = Flags(0x0008) FlagsLiteral = Flags(0x0010) FlagsNumLiteral = Flags(0x0020) FlagsStrLiteral = Flags(0x0040) @@ -42,12 +42,16 @@ const ( // ID combines a Key and Flags. type ID uint32 +func (x ID) UnaryForm() ID { return unaryForms[0xFF&(x>>idShift)] } +func (x ID) BinaryForm() ID { return binaryForms[0xFF&(x>>idShift)] } +func (x ID) AssociativeForm() ID { return associativeForms[0xFF&(x>>idShift)] } + func (x ID) Key() Key { return Key(x >> idShift) } func (x ID) Flags() Flags { return Flags(x & idMask) } func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } -func (x ID) IsAssociative() bool { return Flags(x)&FlagsAssociative != 0 } +func (x ID) IsAssociativeOp() bool { return Flags(x)&FlagsAssociativeOp != 0 } func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 } func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 } @@ -70,7 +74,7 @@ func (t Token) Flags() Flags { return Flags(t.ID & idMask) } func (t Token) IsUnaryOp() bool { return Flags(t.ID)&FlagsUnaryOp != 0 } func (t Token) IsBinaryOp() bool { return Flags(t.ID)&FlagsBinaryOp != 0 } -func (t Token) IsAssociative() bool { return Flags(t.ID)&FlagsAssociative != 0 } +func (t Token) IsAssociativeOp() bool { return Flags(t.ID)&FlagsAssociativeOp != 0 } func (t Token) IsLiteral() bool { return Flags(t.ID)&FlagsLiteral != 0 } func (t Token) IsNumLiteral() bool { return Flags(t.ID)&FlagsNumLiteral != 0 } func (t Token) IsStrLiteral() bool { return Flags(t.ID)&FlagsStrLiteral != 0 } @@ -84,13 +88,13 @@ func (t Token) IsImplicitSemicolon() bool { return Flags(t.ID)&FlagsImplicitSemi // nBuiltInKeys is the number of built-in Keys. The packing is: // - Zero is invalid. -// - [0x01, 0x3F] are squiggly punctuation, such as "(", ")" and ";". -// - [0x40, 0x5F] are squiggly assignments, such as "=" and "+=". -// - [0x60, 0x7F] are squiggly operators, such as "+" and "==". -// - [0x80, 0x9F] are alpha-numeric operators, such as "and" and "as". -// - [0xA0, 0xCF] are keywords, such as "if" and "return". -// - [0xD0, 0xDF] are literals, such as "false" and "true". -// - [0xE0, 0xFF] are identifiers, such as "bool" and "u32". +// - [0x01, 0x2F] are squiggly punctuation, such as "(", ")" and ";". +// - [0x30, 0x3F] are squiggly assignments, such as "=" and "+=". +// - [0x40, 0x5F] are operators, such as "+", "==" and "not". +// - [0x60, 0x8F] are keywords, such as "if" and "return". +// - [0x90, 0x9F] are literals, such as "false" and "true". +// - [0xA0, 0xBF] are identifiers, such as "bool" and "u32". +// - [0xC0, 0xFF] are disambiguation forms, e.g. unary "+" vs binary "+". // // "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and // "&=". Their Keys range in [0x01, 0x7F]. @@ -112,73 +116,112 @@ const ( IDColon = ID(0x23<> idShift: IDXUnaryPlus, + IDMinus >> idShift: IDXUnaryMinus, + IDNot >> idShift: IDXUnaryNot, +} + +var binaryForms = [256]ID{ + IDPlus >> idShift: IDXBinaryPlus, + IDMinus >> idShift: IDXBinaryMinus, + IDStar >> idShift: IDXBinaryStar, + IDSlash >> idShift: IDXBinarySlash, + IDShiftL >> idShift: IDXBinaryShiftL, + IDShiftR >> idShift: IDXBinaryShiftR, + IDAmp >> idShift: IDXBinaryAmp, + IDAmpHat >> idShift: IDXBinaryAmpHat, + IDPipe >> idShift: IDXBinaryPipe, + IDHat >> idShift: IDXBinaryHat, + IDNotEq >> idShift: IDXBinaryNotEq, + IDLessThan >> idShift: IDXBinaryLessThan, + IDLessEq >> idShift: IDXBinaryLessEq, + IDEqEq >> idShift: IDXBinaryEqEq, + IDGreaterEq >> idShift: IDXBinaryGreaterEq, + IDGreaterThan >> idShift: IDXBinaryGreaterThan, + IDAnd >> idShift: IDXBinaryAnd, + IDOr >> idShift: IDXBinaryOr, + IDAs >> idShift: IDXBinaryAs, +} + +var associativeForms = [256]ID{ + IDPlus >> idShift: IDXAssociativePlus, + IDStar >> idShift: IDXAssociativeStar, + IDAmp >> idShift: IDXAssociativeAmp, + IDPipe >> idShift: IDXAssociativePipe, + IDHat >> idShift: IDXAssociativeHat, + IDAnd >> idShift: IDXAssociativeAnd, + IDOr >> idShift: IDXAssociativeOr, +}