From ea2bc6c10cb001ff9e563762eb5a7f556d6ded58 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 29 Apr 2017 16:57:56 +1000 Subject: [PATCH] First cut of a parser. --- lang/ast/ast.go | 85 +++++++++++++++ lang/parse/parse.go | 258 ++++++++++++++++++++++++++++++++++++++++++++ lang/token/list.go | 125 ++++++++++++++++----- lang/token/token.go | 13 --- 4 files changed, 439 insertions(+), 42 deletions(-) create mode 100644 lang/ast/ast.go create mode 100644 lang/parse/parse.go diff --git a/lang/ast/ast.go b/lang/ast/ast.go new file mode 100644 index 00000000..8539a6b1 --- /dev/null +++ b/lang/ast/ast.go @@ -0,0 +1,85 @@ +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +package ast + +import ( + "github.com/google/puffs/lang/token" +) + +// Kind is what kind of node it is. For example, a top-level func or a numeric +// constant. Kind is different from Type; the latter is used for type-checking +// in the programming language sense. +type Kind uint32 + +const ( + KInvalid = Kind(iota) + + // KLiteral is a literal value. + // - ID0: value + KLiteral + + // KIdent is an identifier. + // - ID0: name + KIdent + + // KAssign is "Left = Right". + // - LHS: Left + // - RHS: Right + KAssign + + // KParam is a "name package.type" parameter: + // - ID0: name + // - ID1: package (optional) + // - ID2: type + KParam + + // KFunc is "func ID0.ID1 (List0) (List1) { List2 }": + // - ID0: receiver (optional) + // - ID1: name + // - List0: in-parameters + // - List1: out-parameters + // - List2: function body + // - FlagsSuspendible is (List1) vs ?(List1) + KFunc + + // KFile is a file of source code. + // - List0: top-level declarations + KFile +) + +func (k Kind) String() string { + if uint(k) < uint(len(kindStrings)) { + return kindStrings[k] + } + return "KUnknown" +} + +var kindStrings = [...]string{ + KAssign: "KAssign", + KFile: "KFile", + KFunc: "KFunc", + KIdent: "KIdent", + KInvalid: "KInvalid", + KLiteral: "KLiteral", + KParam: "KParam", +} + +type Flags uint32 + +const ( + FlagsSuspendible = Flags(0x00000001) +) + +type Node struct { + Kind Kind + Flags Flags + ID0 token.ID + ID1 token.ID + ID2 token.ID + LHS *Node // Left Hand Side. + RHS *Node // Right Hand Side. + List0 []*Node + List1 []*Node + List2 []*Node +} diff --git a/lang/parse/parse.go b/lang/parse/parse.go new file mode 100644 index 00000000..66c92215 --- /dev/null +++ b/lang/parse/parse.go @@ -0,0 +1,258 @@ +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +package parse + +import ( + "fmt" + + a "github.com/google/puffs/lang/ast" + t "github.com/google/puffs/lang/token" +) + +func isIdentifier(name string) bool { + if name == "" { + return false + } + // It suffices to check the first byte. A numeric literal will start with + // [0-9]. + // + // TODO: how does this interact with comment tokens? + c := name[0] + return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') +} + +func ParseFile(src []t.Token, m *t.IDMap, filename string) (*a.Node, error) { + p := &parser{ + src: src, + m: m, + filename: filename, + } + if len(src) > 0 { + p.lastLine = src[len(src)-1].Line + } + return p.parseFile() +} + +type parser struct { + src []t.Token + m *t.IDMap + filename string + lastLine int32 +} + +func (p *parser) line() int32 { + if len(p.src) != 0 { + return p.src[0].Line + } + return p.lastLine +} + +func (p *parser) peekID() t.ID { + if len(p.src) > 0 { + return p.src[0].ID + } + return 0 +} + +func (p *parser) parseFile() (*a.Node, error) { + decls := []*a.Node(nil) + for len(p.src) > 0 { + d, err := p.parseTopLevelDecl() + if err != nil { + return nil, err + } + decls = append(decls, d) + } + return &a.Node{ + Kind: a.KFile, + List0: decls, + }, nil +} + +func (p *parser) parseTopLevelDecl() (*a.Node, error) { + switch p.src[0].ID { + case t.IDFunc: + flags := a.Flags(0) + p.src = p.src[1:] + id0, id1, err := p.parseQualifiedIdent() + if err != nil { + return nil, err + } + inParams, err := p.parseParamList() + if err != nil { + return nil, err + } + if p.peekID() == t.IDQuestion { + flags |= a.FlagsSuspendible + p.src = p.src[1:] + } + outParams, err := p.parseParamList() + if err != nil { + return nil, err + } + block, err := p.parseBlock() + if err != nil { + return nil, err + } + if p.peekID() != t.IDSemicolon { + return nil, fmt.Errorf("parse: expected (implicit) ';' at %s:%d", p.filename, p.line()) + } + p.src = p.src[1:] + return &a.Node{ + Kind: a.KFunc, + Flags: flags, + ID0: id0, + ID1: id1, + List0: inParams, + List1: outParams, + List2: block, + }, nil + } + return nil, fmt.Errorf("parse: unrecognized top level declaration at %s:%d", p.filename, p.src[0].Line) +} + +// parseQualifiedIdent parses "foo.bar" or "bar". +func (p *parser) parseQualifiedIdent() (t.ID, t.ID, error) { + x, err := p.parseIdent() + if err != nil { + return 0, 0, err + } + + if p.peekID() != t.IDDot { + return 0, x, nil + } + p.src = p.src[1:] + + y, err := p.parseIdent() + if err != nil { + return 0, 0, err + } + return x, y, nil +} + +func (p *parser) parseIdent() (t.ID, error) { + if len(p.src) == 0 { + return 0, fmt.Errorf("parse: expected identifier at %s:%d", p.filename, p.line()) + } + x := p.src[0] + name := p.m.ByID(x.ID) + if x.ID.IsBuiltIn() || !isIdentifier(name) { + return 0, fmt.Errorf("parse: expected identifier, got %q at %s:%d", name, p.filename, p.line()) + } + p.src = p.src[1:] + return x.ID, nil +} + +func (p *parser) parseParamList() ([]*a.Node, error) { + if p.peekID() != t.IDOpenParen { + return nil, fmt.Errorf("parse: expected '(' for parameter list at %s:%d", p.filename, p.line()) + } + p.src = p.src[1:] + + params := []*a.Node(nil) + for len(p.src) > 0 { + if p.src[0].ID == t.IDCloseParen { + p.src = p.src[1:] + return params, nil + } + + param, err := p.parseParam() + if err != nil { + return nil, err + } + params = append(params, param) + + switch p.peekID() { + case t.IDCloseParen: + p.src = p.src[1:] + return params, nil + case t.IDComma: + p.src = p.src[1:] + default: + return nil, fmt.Errorf("parse: expected ')' for parameter list at %s:%d", p.filename, p.line()) + } + } + return nil, fmt.Errorf("parse: expected ')' for parameter list at %s:%d", p.filename, p.line()) +} + +func (p *parser) parseParam() (*a.Node, error) { + id0, err := p.parseIdent() + if err != nil { + return nil, err + } + id1, id2, err := p.parseQualifiedIdent() + if err != nil { + return nil, err + } + return &a.Node{ + Kind: a.KParam, + ID0: id0, + ID1: id1, + ID2: id2, + }, nil +} + +func (p *parser) parseBlock() ([]*a.Node, error) { + if p.peekID() != t.IDOpenCurly { + return nil, fmt.Errorf("parse: expected '{' for block at %s:%d", p.filename, p.line()) + } + p.src = p.src[1:] + + block := []*a.Node(nil) + for len(p.src) > 0 { + if p.src[0].ID == t.IDCloseCurly { + p.src = p.src[1:] + return block, nil + } + + s, err := p.parseStatement() + if err != nil { + return nil, err + } + block = append(block, s) + + if p.peekID() != t.IDSemicolon { + return nil, fmt.Errorf("parse: expected (implicit) ';' at %s:%d", p.filename, p.line()) + } + p.src = p.src[1:] + } + return nil, fmt.Errorf("parse: expected '}' for block at %s:%d", p.filename, p.line()) +} + +func (p *parser) parseStatement() (*a.Node, error) { + // TODO: parse statements other than x = y. + + lhs, err := p.parseExpr() + if err != nil { + return nil, err + } + + if p.peekID() != t.IDEq { + return nil, fmt.Errorf("parse: expected '=' for statement at %s:%d", p.filename, p.line()) + } + p.src = p.src[1:] + + rhs, err := p.parseExpr() + if err != nil { + return nil, err + } + + return &a.Node{ + Kind: a.KAssign, + LHS: lhs, + RHS: rhs, + }, nil +} + +func (p *parser) parseExpr() (*a.Node, error) { + // TODO: parse other expressions, such as x.y, unop x and x binop y. + id, err := p.parseIdent() + if err != nil { + return nil, err + } + return &a.Node{ + Kind: a.KIdent, + ID0: id, + }, nil +} diff --git a/lang/token/list.go b/lang/token/list.go index 0d79e195..70662fd6 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -3,6 +3,28 @@ package token +// nBuiltIns is the number of built-in IDs. The packing is: +// - Zero is invalid. +// - [0x001, 0x03F] are squiggly punctuation, such as "(", ")" and ";". +// - [0x040, 0x05F] are squiggly assignments, such as "=" and "+=". +// - [0x060, 0x07F] are squiggly operators, such as "+" and "==". +// - [0x080, 0x09F] are alpha-numeric operators, such as "and" and "as". +// - [0x0A0, 0x0CF] are keywords, such as "if" and "return". +// - [0x0D0, 0x0DF] are literals, such as "false" and "true". +// - [0x0E0, 0x0FF] are identifiers, such as "bool" and "u32". +// - [0x100, 0x1FF] aren't returned by Tokenize. The space encodes ambiguous +// 1-byte squiggles. For example, & might be the start of &^ or &=. +// +// "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and +// "&=". Their IDs range in [0x001, 0x07F]. +const nBuiltIns = 512 + +type ID uint32 + +func (x ID) IsBuiltIn() bool { return x < nBuiltIns } + +func (x ID) implicitSemicolon() bool { return x >= nBuiltIns || builtInsImplicitSemicolons[x&0xFF] } + const ( IDInvalid = ID(0x00) @@ -19,17 +41,6 @@ const ( IDColon = ID(0x23) IDSemicolon = ID(0x24) - IDPlus = ID(0x31) - IDMinus = ID(0x32) - IDStar = ID(0x33) - IDSlash = ID(0x34) - IDShiftL = ID(0x35) - IDShiftR = ID(0x36) - IDAmp = ID(0x37) - IDAmpHat = ID(0x38) - IDPipe = ID(0x39) - IDHat = ID(0x3A) - IDEq = ID(0x40) IDPlusEq = ID(0x41) IDMinusEq = ID(0x42) @@ -42,17 +53,50 @@ const ( IDPipeEq = ID(0x49) IDHatEq = ID(0x4A) - IDNotEq = ID(0x50) - IDLessThan = ID(0x51) - IDLessEq = ID(0x52) - IDEqEq = ID(0x53) - IDGreaterEq = ID(0x54) - IDGreaterThan = ID(0x55) + IDPlus = ID(0x61) + IDMinus = ID(0x62) + IDStar = ID(0x63) + IDSlash = ID(0x64) + IDShiftL = ID(0x65) + IDShiftR = ID(0x66) + IDAmp = ID(0x67) + IDAmpHat = ID(0x68) + IDPipe = ID(0x69) + IDHat = ID(0x6A) - // TODO: sort these keywords by name, when the list has stabilized. + IDNotEq = ID(0x70) + IDLessThan = ID(0x71) + IDLessEq = ID(0x72) + IDEqEq = ID(0x73) + IDGreaterEq = ID(0x74) + IDGreaterThan = ID(0x75) + + // TODO: sort these by name, when the list has stabilized. IDAnd = ID(0x80) IDOr = ID(0x81) IDNot = ID(0x82) + IDAs = ID(0x83) + + // TODO: sort these by name, when the list has stabilized. + IDFunc = ID(0xA0) + + IDFalse = ID(0xD0) + IDTrue = ID(0xD1) + + IDI8 = ID(0xE0) + IDI16 = ID(0xE1) + IDI32 = ID(0xE2) + IDI64 = ID(0xE3) + IDU8 = ID(0xE4) + IDU16 = ID(0xE5) + IDU32 = ID(0xE6) + IDU64 = ID(0xE7) + IDUsize = ID(0xE8) + IDBool = ID(0xE9) + IDBuf1 = ID(0xEA) + IDBuf2 = ID(0xEB) + + IDUnderscore = ID(0xFF) lexerBase = ID(0x100) ) @@ -71,17 +115,6 @@ var builtInsByID = [nBuiltIns]string{ IDColon: ":", IDSemicolon: ";", - IDPlus: "+", - IDMinus: "-", - IDStar: "*", - IDSlash: "/", - IDShiftL: "<<", - IDShiftR: ">>", - IDAmp: "&", - IDAmpHat: "&^", - IDPipe: "|", - IDHat: "^", - IDEq: "=", IDPlusEq: "+=", IDMinusEq: "-=", @@ -94,6 +127,17 @@ var builtInsByID = [nBuiltIns]string{ IDPipeEq: "|=", IDHatEq: "^=", + IDPlus: "+", + IDMinus: "-", + IDStar: "*", + IDSlash: "/", + IDShiftL: "<<", + IDShiftR: ">>", + IDAmp: "&", + IDAmpHat: "&^", + IDPipe: "|", + IDHat: "^", + IDNotEq: "!=", IDLessThan: "<", IDLessEq: "<=", @@ -104,6 +148,27 @@ var builtInsByID = [nBuiltIns]string{ IDAnd: "and", IDOr: "or", IDNot: "not", + IDAs: "as", + + IDFunc: "func", + + IDFalse: "false", + IDTrue: "true", + + IDI8: "i8", + IDI16: "i16", + IDI32: "i32", + IDI64: "i64", + IDU8: "u8", + IDU16: "u16", + IDU32: "u32", + IDU64: "u64", + IDUsize: "usize", + IDBool: "bool", + IDBuf1: "buf1", + IDBuf2: "buf2", + + IDUnderscore: "_", } var builtInsByName = map[string]ID{} @@ -198,11 +263,13 @@ var lexers = [256][]suffixLexer{ {"", IDEq}, }, '<': { + {"<=", IDShiftLEq}, {"<", IDShiftL}, {"=", IDLessEq}, {"", IDLessThan}, }, '>': { + {">=", IDShiftREq}, {">", IDShiftR}, {"=", IDGreaterEq}, {"", IDGreaterThan}, diff --git a/lang/token/token.go b/lang/token/token.go index 8e43cd40..825e62ee 100644 --- a/lang/token/token.go +++ b/lang/token/token.go @@ -8,19 +8,6 @@ import ( "fmt" ) -// nBuiltIns is the number of built-in IDs: -// - The first 128 are squiggles, such as operators and parentheses. -// - The next 128 are keywords. -// - The last 256 aren't returned by Tokenize. The space encodes ambiguous -// 1-byte operators. For example, & might be the start of &^ or &=. -const nBuiltIns = 512 - -type ID uint32 - -func (t ID) isBuiltIn() bool { return t < nBuiltIns } - -func (t ID) implicitSemicolon() bool { return t >= nBuiltIns || builtInsImplicitSemicolons[t&0xFF] } - type IDMap struct { byName map[string]ID byID []string