diff --git a/lang/token/list.go b/lang/token/list.go new file mode 100644 index 00000000..974b04f2 --- /dev/null +++ b/lang/token/list.go @@ -0,0 +1,201 @@ +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +package token + +const ( + IDInvalid = ID(0x00) + + IDOpenParen = ID(0x10) + IDCloseParen = ID(0x11) + IDOpenBracket = ID(0x12) + IDCloseBracket = ID(0x13) + IDOpenCurly = ID(0x14) + IDCloseCurly = ID(0x15) + + IDDot = ID(0x20) + IDComma = ID(0x21) + IDQuestion = ID(0x22) + IDColon = ID(0x23) + IDSemicolon = ID(0x24) + + IDPlus = ID(0x31) + IDMinus = ID(0x32) + IDStar = ID(0x33) + IDSlash = ID(0x34) + IDShiftL = ID(0x35) + IDShiftR = ID(0x36) + IDAmp = ID(0x37) + IDAmpHat = ID(0x38) + IDPipe = ID(0x39) + IDHat = ID(0x3A) + + IDEq = ID(0x40) + IDPlusEq = ID(0x41) + IDMinusEq = ID(0x42) + IDStarEq = ID(0x43) + IDSlashEq = ID(0x44) + IDShiftLEq = ID(0x45) + IDShiftREq = ID(0x46) + IDAmpEq = ID(0x47) + IDAmpHatEq = ID(0x48) + IDPipeEq = ID(0x49) + IDHatEq = ID(0x4A) + + IDNotEq = ID(0x50) + IDLessThan = ID(0x51) + IDLessEq = ID(0x52) + IDEqEq = ID(0x53) + IDGreaterEq = ID(0x54) + IDGreaterThan = ID(0x55) + + // TODO: sort these keywords by name, when the list has stabilized. + IDAnd = ID(0x80) + IDOr = ID(0x81) + IDNot = ID(0x82) + + lexerBase = ID(0x100) +) + +var builtInsByID = [nBuiltIns]string{ + IDOpenParen: "(", + IDCloseParen: ")", + IDOpenBracket: "[", + IDCloseBracket: "]", + IDOpenCurly: "{", + IDCloseCurly: "}", + + IDDot: ".", + IDComma: ",", + IDQuestion: "?", + IDColon: ":", + IDSemicolon: ";", + + IDPlus: "+", + IDMinus: "-", + IDStar: "*", + IDSlash: "/", + IDShiftL: "<<", + IDShiftR: ">>", + IDAmp: "&", + IDAmpHat: "&^", + IDPipe: "|", + IDHat: "^", + + IDEq: "=", + IDPlusEq: "+=", + IDMinusEq: "-=", + IDStarEq: "*=", + IDSlashEq: "/=", + IDShiftLEq: "<<=", + IDShiftREq: ">>=", + IDAmpEq: "&=", + IDAmpHatEq: "&^=", + IDPipeEq: "|=", + IDHatEq: "^=", + + IDNotEq: "!=", + IDLessThan: "<", + IDLessEq: "<=", + IDEqEq: "==", + IDGreaterEq: ">=", + IDGreaterThan: ">", + + IDAnd: "and", + IDOr: "or", + IDNot: "not", +} + +var builtInsByName = map[string]ID{} + +func init() { + for i, s := range builtInsByID { + if s != "" { + builtInsByName[s] = ID(i) + } + } +} + +var operators = [256]ID{ + '(': IDOpenParen, + ')': IDCloseParen, + '[': IDOpenBracket, + ']': IDCloseBracket, + '{': IDOpenCurly, + '}': IDCloseCurly, + + '.': IDDot, + ',': IDComma, + '?': IDQuestion, + ':': IDColon, + ';': IDSemicolon, + + '&': lexerBase + '&', + '|': lexerBase + '|', + '^': lexerBase + '^', + '+': lexerBase + '+', + '-': lexerBase + '-', + '*': lexerBase + '*', + '/': lexerBase + '/', + '=': lexerBase + '=', + '<': lexerBase + '<', + '>': lexerBase + '>', +} + +type suffixLexer struct { + suffix string + id ID +} + +// The order of the []suffixLexer elements matters. The first match wins. Since +// we want to lex greedily, longer suffixes should be earlier in the slice. + +var lexers = [256][]suffixLexer{ + '!': { + {"=", IDNotEq}, + }, + '&': { + {"^=", IDAmpHatEq}, + {"^", IDAmpHat}, + {"=", IDAmpEq}, + {"", IDAmp}, + }, + '|': { + {"=", IDPipeEq}, + {"", IDPipe}, + }, + '^': { + {"=", IDHatEq}, + {"", IDHat}, + }, + '+': { + {"=", IDPlusEq}, + {"", IDPlus}, + }, + '-': { + {"=", IDMinusEq}, + {"", IDMinus}, + }, + '*': { + {"=", IDStarEq}, + {"", IDStar}, + }, + '/': { + {"=", IDSlashEq}, + {"", IDSlash}, + }, + '=': { + {"=", IDEqEq}, + {"", IDEq}, + }, + '<': { + {"<", IDShiftL}, + {"=", IDLessEq}, + {"", IDLessThan}, + }, + '>': { + {">", IDShiftR}, + {"=", IDGreaterEq}, + {"", IDGreaterThan}, + }, +} diff --git a/lang/token/token.go b/lang/token/token.go new file mode 100644 index 00000000..c29f26a6 --- /dev/null +++ b/lang/token/token.go @@ -0,0 +1,203 @@ +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +package token + +import ( + "errors" + "fmt" +) + +const nBuiltIns = 512 + +type ID uint32 + +func (t ID) isBuiltIn() bool { return t < nBuiltIns } + +type IDMap struct { + byName map[string]ID + byID []string +} + +func (m *IDMap) insert(name string) (ID, error) { + if name == "" { + return 0, nil + } + if t, ok := builtInsByName[name]; ok { + return t, nil + } + if m.byName == nil { + m.byName = map[string]ID{} + } + if t, ok := m.byName[name]; ok { + return t, nil + } + if uint32(len(m.byID)) == (1<<32)-nBuiltIns { + return 0, errors.New("token: too many distinct tokens") + } + t := nBuiltIns + ID(len(m.byID)) + m.byName[name] = t + m.byID = append(m.byID, name) + return t, nil +} + +func (m *IDMap) ByName(name string) ID { + if m.byName != nil { + return m.byName[name] + } + return 0 +} + +func (m *IDMap) ByID(id ID) string { + if id < nBuiltIns { + return builtInsByID[id] + } + id -= nBuiltIns + if uint(id) < uint(len(m.byID)) { + return m.byID[id] + } + return "" +} + +type Token struct { + ID ID + Line int32 +} + +func alpha(c byte) bool { + return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') +} + +func alphaNumeric(c byte) bool { + return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9') +} + +func numeric(c byte) bool { + return ('0' <= c && c <= '9') +} + +func hasPrefix(a []byte, s string) bool { + if len(s) == 0 { + return true + } + if len(a) < len(s) { + return false + } + a = a[:len(s)] + for i := range a { + if a[i] != s[i] { + return false + } + } + return true +} + +func containingLine(src []byte, i int) []byte { + j := i + for ; j < len(src) && src[j] != '\n'; j++ { + } + for ; i > 0 && src[i-1] != '\n'; i-- { + } + return src[i:j] +} + +func trim(b []byte) []byte { + // Trim the back. + i := len(b) + for ; i > 0 && b[i-1] <= ' '; i-- { + } + b = b[:i] + + // Trim the front. + i = 0 + for ; i < len(b) && b[i] <= ' '; i++ { + } + b = b[i:] + + return b +} + +func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, retErr error) { + line := int32(1) +loop: + for i := 0; i < len(src); { + c := src[i] + + if c <= ' ' { + if c == '\n' { + // TODO: implicit semi-colon insertion a la Go. + line++ + } + i++ + continue + } + + if alpha(c) { + j := i + 1 + for ; j < len(src) && alphaNumeric(src[j]); j++ { + } + id, err := m.insert(string(src[i:j])) + if err != nil { + return nil, err + } + tokens = append(tokens, Token{id, line}) + i = j + continue + } + + if numeric(c) { + // TODO: 0x12 hex and 0b11 binary numbers. + // + // TODO: allow underscores like 0b1000_0000_1111? + // + // TODO: save numbers in a separate map, not the IDMap?? + j := i + 1 + for ; j < len(src) && numeric(src[j]); j++ { + } + id, err := m.insert(string(src[i:j])) + if err != nil { + return nil, err + } + tokens = append(tokens, Token{id, line}) + i = j + continue + } + + if c == '/' && i+1 < len(src) && src[i+1] == '/' { + // TODO: save comments, dont' just skip them. We'll need to render + // them for puffsfmt. + i += 2 + for ; i < len(src) && src[i] != '\n'; i++ { + } + continue + } + + if id := operators[c]; id != 0 { + i++ + if id < lexerBase { + tokens = append(tokens, Token{id, line}) + continue + } + for _, x := range lexers[id&0xFF] { + if hasPrefix(src[i:], x.suffix) { + i += len(x.suffix) + tokens = append(tokens, Token{x.id, line}) + continue loop + } + } + } + + msg := "" + if c <= 0x7F { + msg = fmt.Sprintf("byte '\\x%02X' (%q)", c, c) + } else { + msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c) + } + if filename == "" { + filename = "line" + } + return nil, fmt.Errorf("token: unrecognized %s at %s:%d. line contents: %q", + msg, filename, line, trim(containingLine(src, i))) + } + return tokens, nil +}