First cut of a tokenizer.

This commit is contained in:
Nigel Tao committed 2017-04-29 11:27:38 +10:00
1 parent bd1884bb65
commit a91ae8cdbf
2 files changed
+404

No files matched your search

+201
View File
@@ -0,0 +1,201 @@
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package token
const (
IDInvalid = ID(0x00)
IDOpenParen = ID(0x10)
IDCloseParen = ID(0x11)
IDOpenBracket = ID(0x12)
IDCloseBracket = ID(0x13)
IDOpenCurly = ID(0x14)
IDCloseCurly = ID(0x15)
IDDot = ID(0x20)
IDComma = ID(0x21)
IDQuestion = ID(0x22)
IDColon = ID(0x23)
IDSemicolon = ID(0x24)
IDPlus = ID(0x31)
IDMinus = ID(0x32)
IDStar = ID(0x33)
IDSlash = ID(0x34)
IDShiftL = ID(0x35)
IDShiftR = ID(0x36)
IDAmp = ID(0x37)
IDAmpHat = ID(0x38)
IDPipe = ID(0x39)
IDHat = ID(0x3A)
IDEq = ID(0x40)
IDPlusEq = ID(0x41)
IDMinusEq = ID(0x42)
IDStarEq = ID(0x43)
IDSlashEq = ID(0x44)
IDShiftLEq = ID(0x45)
IDShiftREq = ID(0x46)
IDAmpEq = ID(0x47)
IDAmpHatEq = ID(0x48)
IDPipeEq = ID(0x49)
IDHatEq = ID(0x4A)
IDNotEq = ID(0x50)
IDLessThan = ID(0x51)
IDLessEq = ID(0x52)
IDEqEq = ID(0x53)
IDGreaterEq = ID(0x54)
IDGreaterThan = ID(0x55)
// TODO: sort these keywords by name, when the list has stabilized.
IDAnd = ID(0x80)
IDOr = ID(0x81)
IDNot = ID(0x82)
lexerBase = ID(0x100)
)
var builtInsByID = [nBuiltIns]string{
IDOpenParen: "(",
IDCloseParen: ")",
IDOpenBracket: "[",
IDCloseBracket: "]",
IDOpenCurly: "{",
IDCloseCurly: "}",
IDDot: ".",
IDComma: ",",
IDQuestion: "?",
IDColon: ":",
IDSemicolon: ";",
IDPlus: "+",
IDMinus: "-",
IDStar: "*",
IDSlash: "/",
IDShiftL: "<<",
IDShiftR: ">>",
IDAmp: "&",
IDAmpHat: "&^",
IDPipe: "|",
IDHat: "^",
IDEq: "=",
IDPlusEq: "+=",
IDMinusEq: "-=",
IDStarEq: "*=",
IDSlashEq: "/=",
IDShiftLEq: "<<=",
IDShiftREq: ">>=",
IDAmpEq: "&=",
IDAmpHatEq: "&^=",
IDPipeEq: "|=",
IDHatEq: "^=",
IDNotEq: "!=",
IDLessThan: "<",
IDLessEq: "<=",
IDEqEq: "==",
IDGreaterEq: ">=",
IDGreaterThan: ">",
IDAnd: "and",
IDOr: "or",
IDNot: "not",
}
var builtInsByName = map[string]ID{}
func init() {
for i, s := range builtInsByID {
if s != "" {
builtInsByName[s] = ID(i)
}
}
}
var operators = [256]ID{
'(': IDOpenParen,
')': IDCloseParen,
'[': IDOpenBracket,
']': IDCloseBracket,
'{': IDOpenCurly,
'}': IDCloseCurly,
'.': IDDot,
',': IDComma,
'?': IDQuestion,
':': IDColon,
';': IDSemicolon,
'&': lexerBase + '&',
'|': lexerBase + '|',
'^': lexerBase + '^',
'+': lexerBase + '+',
'-': lexerBase + '-',
'*': lexerBase + '*',
'/': lexerBase + '/',
'=': lexerBase + '=',
'<': lexerBase + '<',
'>': lexerBase + '>',
}
type suffixLexer struct {
suffix string
id ID
}
// The order of the []suffixLexer elements matters. The first match wins. Since
// we want to lex greedily, longer suffixes should be earlier in the slice.
var lexers = [256][]suffixLexer{
'!': {
{"=", IDNotEq},
},
'&': {
{"^=", IDAmpHatEq},
{"^", IDAmpHat},
{"=", IDAmpEq},
{"", IDAmp},
},
'|': {
{"=", IDPipeEq},
{"", IDPipe},
},
'^': {
{"=", IDHatEq},
{"", IDHat},
},
'+': {
{"=", IDPlusEq},
{"", IDPlus},
},
'-': {
{"=", IDMinusEq},
{"", IDMinus},
},
'*': {
{"=", IDStarEq},
{"", IDStar},
},
'/': {
{"=", IDSlashEq},
{"", IDSlash},
},
'=': {
{"=", IDEqEq},
{"", IDEq},
},
'<': {
{"<", IDShiftL},
{"=", IDLessEq},
{"", IDLessThan},
},
'>': {
{">", IDShiftR},
{"=", IDGreaterEq},
{"", IDGreaterThan},
},
}
+203
View File
@@ -0,0 +1,203 @@
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package token
import (
"errors"
"fmt"
)
const nBuiltIns = 512
type ID uint32
func (t ID) isBuiltIn() bool { return t < nBuiltIns }
type IDMap struct {
byName map[string]ID
byID []string
}
func (m *IDMap) insert(name string) (ID, error) {
if name == "" {
return 0, nil
}
if t, ok := builtInsByName[name]; ok {
return t, nil
}
if m.byName == nil {
m.byName = map[string]ID{}
}
if t, ok := m.byName[name]; ok {
return t, nil
}
if uint32(len(m.byID)) == (1<<32)-nBuiltIns {
return 0, errors.New("token: too many distinct tokens")
}
t := nBuiltIns + ID(len(m.byID))
m.byName[name] = t
m.byID = append(m.byID, name)
return t, nil
}
func (m *IDMap) ByName(name string) ID {
if m.byName != nil {
return m.byName[name]
}
return 0
}
func (m *IDMap) ByID(id ID) string {
if id < nBuiltIns {
return builtInsByID[id]
}
id -= nBuiltIns
if uint(id) < uint(len(m.byID)) {
return m.byID[id]
}
return ""
}
type Token struct {
ID ID
Line int32
}
func alpha(c byte) bool {
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_')
}
func alphaNumeric(c byte) bool {
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9')
}
func numeric(c byte) bool {
return ('0' <= c && c <= '9')
}
func hasPrefix(a []byte, s string) bool {
if len(s) == 0 {
return true
}
if len(a) < len(s) {
return false
}
a = a[:len(s)]
for i := range a {
if a[i] != s[i] {
return false
}
}
return true
}
func containingLine(src []byte, i int) []byte {
j := i
for ; j < len(src) && src[j] != '\n'; j++ {
}
for ; i > 0 && src[i-1] != '\n'; i-- {
}
return src[i:j]
}
func trim(b []byte) []byte {
// Trim the back.
i := len(b)
for ; i > 0 && b[i-1] <= ' '; i-- {
}
b = b[:i]
// Trim the front.
i = 0
for ; i < len(b) && b[i] <= ' '; i++ {
}
b = b[i:]
return b
}
func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, retErr error) {
line := int32(1)
loop:
for i := 0; i < len(src); {
c := src[i]
if c <= ' ' {
if c == '\n' {
// TODO: implicit semi-colon insertion a la Go.
line++
}
i++
continue
}
if alpha(c) {
j := i + 1
for ; j < len(src) && alphaNumeric(src[j]); j++ {
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, err
}
tokens = append(tokens, Token{id, line})
i = j
continue
}
if numeric(c) {
// TODO: 0x12 hex and 0b11 binary numbers.
//
// TODO: allow underscores like 0b1000_0000_1111?
//
// TODO: save numbers in a separate map, not the IDMap??
j := i + 1
for ; j < len(src) && numeric(src[j]); j++ {
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, err
}
tokens = append(tokens, Token{id, line})
i = j
continue
}
if c == '/' && i+1 < len(src) && src[i+1] == '/' {
// TODO: save comments, dont' just skip them. We'll need to render
// them for puffsfmt.
i += 2
for ; i < len(src) && src[i] != '\n'; i++ {
}
continue
}
if id := operators[c]; id != 0 {
i++
if id < lexerBase {
tokens = append(tokens, Token{id, line})
continue
}
for _, x := range lexers[id&0xFF] {
if hasPrefix(src[i:], x.suffix) {
i += len(x.suffix)
tokens = append(tokens, Token{x.id, line})
continue loop
}
}
}
msg := ""
if c <= 0x7F {
msg = fmt.Sprintf("byte '\\x%02X' (%q)", c, c)
} else {
msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c)
}
if filename == "" {
filename = "line"
}
return nil, fmt.Errorf("token: unrecognized %s at %s:%d. line contents: %q",
msg, filename, line, trim(containingLine(src, i)))
}
return tokens, nil
}