diff --git a/lang/parse/parse.go b/lang/parse/parse.go index 66c92215..ab875b32 100644 --- a/lang/parse/parse.go +++ b/lang/parse/parse.go @@ -10,18 +10,6 @@ import ( t "github.com/google/puffs/lang/token" ) -func isIdentifier(name string) bool { - if name == "" { - return false - } - // It suffices to check the first byte. A numeric literal will start with - // [0-9]. - // - // TODO: how does this interact with comment tokens? - c := name[0] - return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') -} - func ParseFile(src []t.Token, m *t.IDMap, filename string) (*a.Node, error) { p := &parser{ src: src, @@ -136,8 +124,8 @@ func (p *parser) parseIdent() (t.ID, error) { return 0, fmt.Errorf("parse: expected identifier at %s:%d", p.filename, p.line()) } x := p.src[0] - name := p.m.ByID(x.ID) - if x.ID.IsBuiltIn() || !isIdentifier(name) { + if !x.ID.IsIdent() { + name := p.m.ByKey(x.ID.Key()) return 0, fmt.Errorf("parse: expected identifier, got %q at %s:%d", name, p.filename, p.line()) } p.src = p.src[1:] diff --git a/lang/token/list.go b/lang/token/list.go index 70662fd6..9c06aae1 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -3,193 +3,224 @@ package token -// nBuiltIns is the number of built-in IDs. The packing is: -// - Zero is invalid. -// - [0x001, 0x03F] are squiggly punctuation, such as "(", ")" and ";". -// - [0x040, 0x05F] are squiggly assignments, such as "=" and "+=". -// - [0x060, 0x07F] are squiggly operators, such as "+" and "==". -// - [0x080, 0x09F] are alpha-numeric operators, such as "and" and "as". -// - [0x0A0, 0x0CF] are keywords, such as "if" and "return". -// - [0x0D0, 0x0DF] are literals, such as "false" and "true". -// - [0x0E0, 0x0FF] are identifiers, such as "bool" and "u32". -// - [0x100, 0x1FF] aren't returned by Tokenize. The space encodes ambiguous -// 1-byte squiggles. For example, & might be the start of &^ or &=. +// Flags are the low 8 bits of an ID. For example, they signify whether a token +// is an operator, an identifier, etc. // -// "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and -// "&=". Their IDs range in [0x001, 0x07F]. -const nBuiltIns = 512 - -type ID uint32 - -func (x ID) IsBuiltIn() bool { return x < nBuiltIns } - -func (x ID) implicitSemicolon() bool { return x >= nBuiltIns || builtInsImplicitSemicolons[x&0xFF] } +// The flags are not exclusive. For example, the "+" token is both a unary and +// binary operator. It is also associative: (a + b) + c equals a + (b + c). +// +// A valid token will have non-zero flags. If none of the other flags apply, +// the FlagsOther bit will be set. +type Flags uint32 const ( - IDInvalid = ID(0x00) - - IDOpenParen = ID(0x10) - IDCloseParen = ID(0x11) - IDOpenBracket = ID(0x12) - IDCloseBracket = ID(0x13) - IDOpenCurly = ID(0x14) - IDCloseCurly = ID(0x15) - - IDDot = ID(0x20) - IDComma = ID(0x21) - IDQuestion = ID(0x22) - IDColon = ID(0x23) - IDSemicolon = ID(0x24) - - IDEq = ID(0x40) - IDPlusEq = ID(0x41) - IDMinusEq = ID(0x42) - IDStarEq = ID(0x43) - IDSlashEq = ID(0x44) - IDShiftLEq = ID(0x45) - IDShiftREq = ID(0x46) - IDAmpEq = ID(0x47) - IDAmpHatEq = ID(0x48) - IDPipeEq = ID(0x49) - IDHatEq = ID(0x4A) - - IDPlus = ID(0x61) - IDMinus = ID(0x62) - IDStar = ID(0x63) - IDSlash = ID(0x64) - IDShiftL = ID(0x65) - IDShiftR = ID(0x66) - IDAmp = ID(0x67) - IDAmpHat = ID(0x68) - IDPipe = ID(0x69) - IDHat = ID(0x6A) - - IDNotEq = ID(0x70) - IDLessThan = ID(0x71) - IDLessEq = ID(0x72) - IDEqEq = ID(0x73) - IDGreaterEq = ID(0x74) - IDGreaterThan = ID(0x75) - - // TODO: sort these by name, when the list has stabilized. - IDAnd = ID(0x80) - IDOr = ID(0x81) - IDNot = ID(0x82) - IDAs = ID(0x83) - - // TODO: sort these by name, when the list has stabilized. - IDFunc = ID(0xA0) - - IDFalse = ID(0xD0) - IDTrue = ID(0xD1) - - IDI8 = ID(0xE0) - IDI16 = ID(0xE1) - IDI32 = ID(0xE2) - IDI64 = ID(0xE3) - IDU8 = ID(0xE4) - IDU16 = ID(0xE5) - IDU32 = ID(0xE6) - IDU64 = ID(0xE7) - IDUsize = ID(0xE8) - IDBool = ID(0xE9) - IDBuf1 = ID(0xEA) - IDBuf2 = ID(0xEB) - - IDUnderscore = ID(0xFF) - - lexerBase = ID(0x100) + FlagsOther = Flags(0x01) + FlagsUnaryOp = Flags(0x02) + FlagsBinaryOp = Flags(0x04) + FlagsAssociative = Flags(0x08) + FlagsAssign = Flags(0x10) + FlagsLiteral = Flags(0x20) + FlagsIdent = Flags(0x40) + FlagsImplicitSemicolon = Flags(0x80) ) -var builtInsByID = [nBuiltIns]string{ - IDOpenParen: "(", - IDCloseParen: ")", - IDOpenBracket: "[", - IDCloseBracket: "]", - IDOpenCurly: "{", - IDCloseCurly: "}", +// Key is the high 24 bits of an ID. It is the map key for an IDMap. +type Key uint32 - IDDot: ".", - IDComma: ",", - IDQuestion: "?", - IDColon: ":", - IDSemicolon: ";", +const ( + idShift = 8 + idMask = 1<<8 - 1 + maxKey = 1<<24 - 1 +) - IDEq: "=", - IDPlusEq: "+=", - IDMinusEq: "-=", - IDStarEq: "*=", - IDSlashEq: "/=", - IDShiftLEq: "<<=", - IDShiftREq: ">>=", - IDAmpEq: "&=", - IDAmpHatEq: "&^=", - IDPipeEq: "|=", - IDHatEq: "^=", +// ID combines a Key and Flags. +type ID uint32 - IDPlus: "+", - IDMinus: "-", - IDStar: "*", - IDSlash: "/", - IDShiftL: "<<", - IDShiftR: ">>", - IDAmp: "&", - IDAmpHat: "&^", - IDPipe: "|", - IDHat: "^", +func (x ID) Key() Key { return Key(x >> idShift) } +func (x ID) Flags() Flags { return Flags(x & idMask) } - IDNotEq: "!=", - IDLessThan: "<", - IDLessEq: "<=", - IDEqEq: "==", - IDGreaterEq: ">=", - IDGreaterThan: ">", +func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } +func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } +func (x ID) IsAssociative() bool { return Flags(x)&FlagsAssociative != 0 } +func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 } +func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } +func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 } +func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 } - IDAnd: "and", - IDOr: "or", - IDNot: "not", - IDAs: "as", +// nBuiltInKeys is the number of built-in Keys. The packing is: +// - Zero is invalid. +// - [0x01, 0x3F] are squiggly punctuation, such as "(", ")" and ";". +// - [0x40, 0x5F] are squiggly assignments, such as "=" and "+=". +// - [0x60, 0x7F] are squiggly operators, such as "+" and "==". +// - [0x80, 0x9F] are alpha-numeric operators, such as "and" and "as". +// - [0xA0, 0xCF] are keywords, such as "if" and "return". +// - [0xD0, 0xDF] are literals, such as "false" and "true". +// - [0xE0, 0xFF] are identifiers, such as "bool" and "u32". +// +// "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and +// "&=". Their Keys range in [0x01, 0x7F]. +const nBuiltInKeys = 256 - IDFunc: "func", +const ( + IDInvalid = ID(0) - IDFalse: "false", - IDTrue: "true", + IDOpenParen = ID(0x10<> idShift: {"(", IDOpenParen}, + IDCloseParen >> idShift: {")", IDCloseParen}, + IDOpenBracket >> idShift: {"[", IDOpenBracket}, + IDCloseBracket >> idShift: {"]", IDCloseBracket}, + IDOpenCurly >> idShift: {"{", IDOpenCurly}, + IDCloseCurly >> idShift: {"}", IDCloseCurly}, + + IDDot >> idShift: {".", IDDot}, + IDComma >> idShift: {",", IDComma}, + IDQuestion >> idShift: {"?", IDQuestion}, + IDColon >> idShift: {":", IDColon}, + IDSemicolon >> idShift: {";", IDSemicolon}, + + IDEq >> idShift: {"=", IDEq}, + IDPlusEq >> idShift: {"+=", IDPlusEq}, + IDMinusEq >> idShift: {"-=", IDMinusEq}, + IDStarEq >> idShift: {"*=", IDStarEq}, + IDSlashEq >> idShift: {"/=", IDSlashEq}, + IDShiftLEq >> idShift: {"<<=", IDShiftLEq}, + IDShiftREq >> idShift: {">>=", IDShiftREq}, + IDAmpEq >> idShift: {"&=", IDAmpEq}, + IDAmpHatEq >> idShift: {"&^=", IDAmpHatEq}, + IDPipeEq >> idShift: {"|=", IDPipeEq}, + IDHatEq >> idShift: {"^=", IDHatEq}, + + IDPlus >> idShift: {"+", IDPlus}, + IDMinus >> idShift: {"-", IDMinus}, + IDStar >> idShift: {"*", IDStar}, + IDSlash >> idShift: {"/", IDSlash}, + IDShiftL >> idShift: {"<<", IDShiftL}, + IDShiftR >> idShift: {">>", IDShiftR}, + IDAmp >> idShift: {"&", IDAmp}, + IDAmpHat >> idShift: {"&^", IDAmpHat}, + IDPipe >> idShift: {"|", IDPipe}, + IDHat >> idShift: {"^", IDHat}, + + IDNotEq >> idShift: {"!=", IDNotEq}, + IDLessThan >> idShift: {"<", IDLessThan}, + IDLessEq >> idShift: {"<=", IDLessEq}, + IDEqEq >> idShift: {"==", IDEqEq}, + IDGreaterEq >> idShift: {">=", IDGreaterEq}, + IDGreaterThan >> idShift: {">", IDGreaterThan}, + + IDAnd >> idShift: {"and", IDAnd}, + IDOr >> idShift: {"or", IDOr}, + IDNot >> idShift: {"not", IDNot}, + IDAs >> idShift: {"as", IDAs}, + + IDFunc >> idShift: {"func", IDFunc}, + + IDFalse >> idShift: {"false", IDFalse}, + IDTrue >> idShift: {"true", IDTrue}, + + IDI8 >> idShift: {"i8", IDI8}, + IDI16 >> idShift: {"i16", IDI16}, + IDI32 >> idShift: {"i32", IDI32}, + IDI64 >> idShift: {"i64", IDI64}, + IDU8 >> idShift: {"u8", IDU8}, + IDU16 >> idShift: {"u16", IDU16}, + IDU32 >> idShift: {"u32", IDU32}, + IDU64 >> idShift: {"u64", IDU64}, + IDUsize >> idShift: {"usize", IDUsize}, + IDBool >> idShift: {"bool", IDBool}, + IDBuf1 >> idShift: {"buf1", IDBuf1}, + IDBuf2 >> idShift: {"buf2", IDBuf2}, + + IDUnderscore >> idShift: {"_", IDUnderscore}, + IDThis >> idShift: {"this", IDThis}, } var builtInsByName = map[string]ID{} func init() { - for i, s := range builtInsByID { - if s != "" { - builtInsByName[s] = ID(i) + for _, x := range builtInsByKey { + if x.name != "" { + builtInsByName[x.name] = x.id } } } -var builtInsImplicitSemicolons = [256]bool{ - IDCloseParen: true, - IDCloseBracket: true, - IDCloseCurly: true, - - // TODO: break, continue, fallthrough, return, a la Go. -} - -// squiggles are built-in tokens that aren't alpha-numeric. +// squiggles are built-in IDs that aren't alpha-numeric. var squiggles = [256]ID{ '(': IDOpenParen, ')': IDCloseParen, @@ -204,16 +235,19 @@ var squiggles = [256]ID{ ':': IDColon, ';': IDSemicolon, - '&': lexerBase + '&', - '|': lexerBase + '|', - '^': lexerBase + '^', - '+': lexerBase + '+', - '-': lexerBase + '-', - '*': lexerBase + '*', - '/': lexerBase + '/', - '=': lexerBase + '=', - '<': lexerBase + '<', - '>': lexerBase + '>', + // 1<': 1 << idShift, } type suffixLexer struct { @@ -221,9 +255,11 @@ type suffixLexer struct { id ID } +// lexers lex ambiguous 1-byte squiggles. For example, "&" might be the start +// of "&^" or "&=". +// // The order of the []suffixLexer elements matters. The first match wins. Since // we want to lex greedily, longer suffixes should be earlier in the slice. - var lexers = [256][]suffixLexer{ '!': { {"=", IDNotEq}, diff --git a/lang/token/token.go b/lang/token/token.go index 825e62ee..a9347162 100644 --- a/lang/token/token.go +++ b/lang/token/token.go @@ -10,29 +10,37 @@ import ( type IDMap struct { byName map[string]ID - byID []string + byKey []string } func (m *IDMap) insert(name string) (ID, error) { if name == "" { return 0, nil } - if t, ok := builtInsByName[name]; ok { - return t, nil + if id, ok := builtInsByName[name]; ok { + return id, nil } if m.byName == nil { m.byName = map[string]ID{} } - if t, ok := m.byName[name]; ok { - return t, nil + if id, ok := m.byName[name]; ok { + return id, nil } - if uint32(len(m.byID)) == (1<<32)-nBuiltIns { + + key := nBuiltInKeys + Key(len(m.byKey)) + if key > maxKey { return 0, errors.New("token: too many distinct tokens") } - t := nBuiltIns + ID(len(m.byID)) - m.byName[name] = t - m.byID = append(m.byID, name) - return t, nil + flags := FlagsImplicitSemicolon + if numeric(name[0]) { + flags |= FlagsLiteral + } else { + flags |= FlagsIdent + } + id := ID(key< 0 && tokens[len(tokens)-1].ID.implicitSemicolon() { + if len(tokens) > 0 && tokens[len(tokens)-1].ID.IsImplicitSemicolon() { tokens = append(tokens, Token{IDSemicolon, line}) } line++ @@ -170,11 +178,11 @@ loop: if id := squiggles[c]; id != 0 { i++ - if id < lexerBase { + if id.Flags() != 0 { tokens = append(tokens, Token{id, line}) continue } - for _, x := range lexers[id&0xFF] { + for _, x := range lexers[c] { if hasPrefix(src[i:], x.suffix) { i += len(x.suffix) tokens = append(tokens, Token{x.id, line})