First cut of a parser.

This commit is contained in:
Nigel Tao committed 2017-04-29 16:57:56 +10:00
1 parent 21c6c34d92
commit ea2bc6c10c
4 files changed
+439 -42

No files matched your search

+85
View File
@@ -0,0 +1,85 @@
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package ast
import (
"github.com/google/puffs/lang/token"
)
// Kind is what kind of node it is. For example, a top-level func or a numeric
// constant. Kind is different from Type; the latter is used for type-checking
// in the programming language sense.
type Kind uint32
const (
KInvalid = Kind(iota)
// KLiteral is a literal value.
// - ID0: value
KLiteral
// KIdent is an identifier.
// - ID0: name
KIdent
// KAssign is "Left = Right".
// - LHS: Left
// - RHS: Right
KAssign
// KParam is a "name package.type" parameter:
// - ID0: name
// - ID1: package (optional)
// - ID2: type
KParam
// KFunc is "func ID0.ID1 (List0) (List1) { List2 }":
// - ID0: receiver (optional)
// - ID1: name
// - List0: <KParam> in-parameters
// - List1: <KParam> out-parameters
// - List2: <KAssign> function body
// - FlagsSuspendible is (List1) vs ?(List1)
KFunc
// KFile is a file of source code.
// - List0: <KFunc> top-level declarations
KFile
)
func (k Kind) String() string {
if uint(k) < uint(len(kindStrings)) {
return kindStrings[k]
}
return "KUnknown"
}
var kindStrings = [...]string{
KAssign: "KAssign",
KFile: "KFile",
KFunc: "KFunc",
KIdent: "KIdent",
KInvalid: "KInvalid",
KLiteral: "KLiteral",
KParam: "KParam",
}
type Flags uint32
const (
FlagsSuspendible = Flags(0x00000001)
)
type Node struct {
Kind Kind
Flags Flags
ID0 token.ID
ID1 token.ID
ID2 token.ID
LHS *Node // Left Hand Side.
RHS *Node // Right Hand Side.
List0 []*Node
List1 []*Node
List2 []*Node
}
+258
View File
@@ -0,0 +1,258 @@
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package parse
import (
"fmt"
a "github.com/google/puffs/lang/ast"
t "github.com/google/puffs/lang/token"
)
func isIdentifier(name string) bool {
if name == "" {
return false
}
// It suffices to check the first byte. A numeric literal will start with
// [0-9].
//
// TODO: how does this interact with comment tokens?
c := name[0]
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_')
}
func ParseFile(src []t.Token, m *t.IDMap, filename string) (*a.Node, error) {
p := &parser{
src: src,
m: m,
filename: filename,
}
if len(src) > 0 {
p.lastLine = src[len(src)-1].Line
}
return p.parseFile()
}
type parser struct {
src []t.Token
m *t.IDMap
filename string
lastLine int32
}
func (p *parser) line() int32 {
if len(p.src) != 0 {
return p.src[0].Line
}
return p.lastLine
}
func (p *parser) peekID() t.ID {
if len(p.src) > 0 {
return p.src[0].ID
}
return 0
}
func (p *parser) parseFile() (*a.Node, error) {
decls := []*a.Node(nil)
for len(p.src) > 0 {
d, err := p.parseTopLevelDecl()
if err != nil {
return nil, err
}
decls = append(decls, d)
}
return &a.Node{
Kind: a.KFile,
List0: decls,
}, nil
}
func (p *parser) parseTopLevelDecl() (*a.Node, error) {
switch p.src[0].ID {
case t.IDFunc:
flags := a.Flags(0)
p.src = p.src[1:]
id0, id1, err := p.parseQualifiedIdent()
if err != nil {
return nil, err
}
inParams, err := p.parseParamList()
if err != nil {
return nil, err
}
if p.peekID() == t.IDQuestion {
flags |= a.FlagsSuspendible
p.src = p.src[1:]
}
outParams, err := p.parseParamList()
if err != nil {
return nil, err
}
block, err := p.parseBlock()
if err != nil {
return nil, err
}
if p.peekID() != t.IDSemicolon {
return nil, fmt.Errorf("parse: expected (implicit) ';' at %s:%d", p.filename, p.line())
}
p.src = p.src[1:]
return &a.Node{
Kind: a.KFunc,
Flags: flags,
ID0: id0,
ID1: id1,
List0: inParams,
List1: outParams,
List2: block,
}, nil
}
return nil, fmt.Errorf("parse: unrecognized top level declaration at %s:%d", p.filename, p.src[0].Line)
}
// parseQualifiedIdent parses "foo.bar" or "bar".
func (p *parser) parseQualifiedIdent() (t.ID, t.ID, error) {
x, err := p.parseIdent()
if err != nil {
return 0, 0, err
}
if p.peekID() != t.IDDot {
return 0, x, nil
}
p.src = p.src[1:]
y, err := p.parseIdent()
if err != nil {
return 0, 0, err
}
return x, y, nil
}
func (p *parser) parseIdent() (t.ID, error) {
if len(p.src) == 0 {
return 0, fmt.Errorf("parse: expected identifier at %s:%d", p.filename, p.line())
}
x := p.src[0]
name := p.m.ByID(x.ID)
if x.ID.IsBuiltIn() || !isIdentifier(name) {
return 0, fmt.Errorf("parse: expected identifier, got %q at %s:%d", name, p.filename, p.line())
}
p.src = p.src[1:]
return x.ID, nil
}
func (p *parser) parseParamList() ([]*a.Node, error) {
if p.peekID() != t.IDOpenParen {
return nil, fmt.Errorf("parse: expected '(' for parameter list at %s:%d", p.filename, p.line())
}
p.src = p.src[1:]
params := []*a.Node(nil)
for len(p.src) > 0 {
if p.src[0].ID == t.IDCloseParen {
p.src = p.src[1:]
return params, nil
}
param, err := p.parseParam()
if err != nil {
return nil, err
}
params = append(params, param)
switch p.peekID() {
case t.IDCloseParen:
p.src = p.src[1:]
return params, nil
case t.IDComma:
p.src = p.src[1:]
default:
return nil, fmt.Errorf("parse: expected ')' for parameter list at %s:%d", p.filename, p.line())
}
}
return nil, fmt.Errorf("parse: expected ')' for parameter list at %s:%d", p.filename, p.line())
}
func (p *parser) parseParam() (*a.Node, error) {
id0, err := p.parseIdent()
if err != nil {
return nil, err
}
id1, id2, err := p.parseQualifiedIdent()
if err != nil {
return nil, err
}
return &a.Node{
Kind: a.KParam,
ID0: id0,
ID1: id1,
ID2: id2,
}, nil
}
func (p *parser) parseBlock() ([]*a.Node, error) {
if p.peekID() != t.IDOpenCurly {
return nil, fmt.Errorf("parse: expected '{' for block at %s:%d", p.filename, p.line())
}
p.src = p.src[1:]
block := []*a.Node(nil)
for len(p.src) > 0 {
if p.src[0].ID == t.IDCloseCurly {
p.src = p.src[1:]
return block, nil
}
s, err := p.parseStatement()
if err != nil {
return nil, err
}
block = append(block, s)
if p.peekID() != t.IDSemicolon {
return nil, fmt.Errorf("parse: expected (implicit) ';' at %s:%d", p.filename, p.line())
}
p.src = p.src[1:]
}
return nil, fmt.Errorf("parse: expected '}' for block at %s:%d", p.filename, p.line())
}
func (p *parser) parseStatement() (*a.Node, error) {
// TODO: parse statements other than x = y.
lhs, err := p.parseExpr()
if err != nil {
return nil, err
}
if p.peekID() != t.IDEq {
return nil, fmt.Errorf("parse: expected '=' for statement at %s:%d", p.filename, p.line())
}
p.src = p.src[1:]
rhs, err := p.parseExpr()
if err != nil {
return nil, err
}
return &a.Node{
Kind: a.KAssign,
LHS: lhs,
RHS: rhs,
}, nil
}
func (p *parser) parseExpr() (*a.Node, error) {
// TODO: parse other expressions, such as x.y, unop x and x binop y.
id, err := p.parseIdent()
if err != nil {
return nil, err
}
return &a.Node{
Kind: a.KIdent,
ID0: id,
}, nil
}
+96 -29
View File
@@ -3,6 +3,28 @@
package token
// nBuiltIns is the number of built-in IDs. The packing is:
// - Zero is invalid.
// - [0x001, 0x03F] are squiggly punctuation, such as "(", ")" and ";".
// - [0x040, 0x05F] are squiggly assignments, such as "=" and "+=".
// - [0x060, 0x07F] are squiggly operators, such as "+" and "==".
// - [0x080, 0x09F] are alpha-numeric operators, such as "and" and "as".
// - [0x0A0, 0x0CF] are keywords, such as "if" and "return".
// - [0x0D0, 0x0DF] are literals, such as "false" and "true".
// - [0x0E0, 0x0FF] are identifiers, such as "bool" and "u32".
// - [0x100, 0x1FF] aren't returned by Tokenize. The space encodes ambiguous
// 1-byte squiggles. For example, & might be the start of &^ or &=.
//
// "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and
// "&=". Their IDs range in [0x001, 0x07F].
const nBuiltIns = 512
type ID uint32
func (x ID) IsBuiltIn() bool { return x < nBuiltIns }
func (x ID) implicitSemicolon() bool { return x >= nBuiltIns || builtInsImplicitSemicolons[x&0xFF] }
const (
IDInvalid = ID(0x00)
@@ -19,17 +41,6 @@ const (
IDColon = ID(0x23)
IDSemicolon = ID(0x24)
IDPlus = ID(0x31)
IDMinus = ID(0x32)
IDStar = ID(0x33)
IDSlash = ID(0x34)
IDShiftL = ID(0x35)
IDShiftR = ID(0x36)
IDAmp = ID(0x37)
IDAmpHat = ID(0x38)
IDPipe = ID(0x39)
IDHat = ID(0x3A)
IDEq = ID(0x40)
IDPlusEq = ID(0x41)
IDMinusEq = ID(0x42)
@@ -42,17 +53,50 @@ const (
IDPipeEq = ID(0x49)
IDHatEq = ID(0x4A)
IDNotEq = ID(0x50)
IDLessThan = ID(0x51)
IDLessEq = ID(0x52)
IDEqEq = ID(0x53)
IDGreaterEq = ID(0x54)
IDGreaterThan = ID(0x55)
IDPlus = ID(0x61)
IDMinus = ID(0x62)
IDStar = ID(0x63)
IDSlash = ID(0x64)
IDShiftL = ID(0x65)
IDShiftR = ID(0x66)
IDAmp = ID(0x67)
IDAmpHat = ID(0x68)
IDPipe = ID(0x69)
IDHat = ID(0x6A)
// TODO: sort these keywords by name, when the list has stabilized.
IDNotEq = ID(0x70)
IDLessThan = ID(0x71)
IDLessEq = ID(0x72)
IDEqEq = ID(0x73)
IDGreaterEq = ID(0x74)
IDGreaterThan = ID(0x75)
// TODO: sort these by name, when the list has stabilized.
IDAnd = ID(0x80)
IDOr = ID(0x81)
IDNot = ID(0x82)
IDAs = ID(0x83)
// TODO: sort these by name, when the list has stabilized.
IDFunc = ID(0xA0)
IDFalse = ID(0xD0)
IDTrue = ID(0xD1)
IDI8 = ID(0xE0)
IDI16 = ID(0xE1)
IDI32 = ID(0xE2)
IDI64 = ID(0xE3)
IDU8 = ID(0xE4)
IDU16 = ID(0xE5)
IDU32 = ID(0xE6)
IDU64 = ID(0xE7)
IDUsize = ID(0xE8)
IDBool = ID(0xE9)
IDBuf1 = ID(0xEA)
IDBuf2 = ID(0xEB)
IDUnderscore = ID(0xFF)
lexerBase = ID(0x100)
)
@@ -71,17 +115,6 @@ var builtInsByID = [nBuiltIns]string{
IDColon: ":",
IDSemicolon: ";",
IDPlus: "+",
IDMinus: "-",
IDStar: "*",
IDSlash: "/",
IDShiftL: "<<",
IDShiftR: ">>",
IDAmp: "&",
IDAmpHat: "&^",
IDPipe: "|",
IDHat: "^",
IDEq: "=",
IDPlusEq: "+=",
IDMinusEq: "-=",
@@ -94,6 +127,17 @@ var builtInsByID = [nBuiltIns]string{
IDPipeEq: "|=",
IDHatEq: "^=",
IDPlus: "+",
IDMinus: "-",
IDStar: "*",
IDSlash: "/",
IDShiftL: "<<",
IDShiftR: ">>",
IDAmp: "&",
IDAmpHat: "&^",
IDPipe: "|",
IDHat: "^",
IDNotEq: "!=",
IDLessThan: "<",
IDLessEq: "<=",
@@ -104,6 +148,27 @@ var builtInsByID = [nBuiltIns]string{
IDAnd: "and",
IDOr: "or",
IDNot: "not",
IDAs: "as",
IDFunc: "func",
IDFalse: "false",
IDTrue: "true",
IDI8: "i8",
IDI16: "i16",
IDI32: "i32",
IDI64: "i64",
IDU8: "u8",
IDU16: "u16",
IDU32: "u32",
IDU64: "u64",
IDUsize: "usize",
IDBool: "bool",
IDBuf1: "buf1",
IDBuf2: "buf2",
IDUnderscore: "_",
}
var builtInsByName = map[string]ID{}
@@ -198,11 +263,13 @@ var lexers = [256][]suffixLexer{
{"", IDEq},
},
'<': {
{"<=", IDShiftLEq},
{"<", IDShiftL},
{"=", IDLessEq},
{"", IDLessThan},
},
'>': {
{">=", IDShiftREq},
{">", IDShiftR},
{"=", IDGreaterEq},
{"", IDGreaterThan},
-13
View File
@@ -8,19 +8,6 @@ import (
"fmt"
)
// nBuiltIns is the number of built-in IDs:
// - The first 128 are squiggles, such as operators and parentheses.
// - The next 128 are keywords.
// - The last 256 aren't returned by Tokenize. The space encodes ambiguous
// 1-byte operators. For example, & might be the start of &^ or &=.
const nBuiltIns = 512
type ID uint32
func (t ID) isBuiltIn() bool { return t < nBuiltIns }
func (t ID) implicitSemicolon() bool { return t >= nBuiltIns || builtInsImplicitSemicolons[t&0xFF] }
type IDMap struct {
byName map[string]ID
byID []string