Files
wuffs/lang/token/token.go
T

231 lines
5.0 KiB
Go

// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package token
import (
"errors"
"fmt"
)
const (
maxLine = 1048575
maxTokenSize = 1023
)
type Map struct {
byName map[string]ID
byKey []string
}
func (m *Map) insert(name string) (ID, error) {
if name == "" {
return 0, nil
}
if id, ok := builtInsByName[name]; ok {
return id, nil
}
if m.byName == nil {
m.byName = map[string]ID{}
}
if id, ok := m.byName[name]; ok {
return id, nil
}
key := nBuiltInKeys + Key(len(m.byKey))
if key > maxKey {
return 0, errors.New("token: too many distinct tokens")
}
flags := FlagsImplicitSemicolon
if numeric(name[0]) {
flags |= FlagsLiteral | FlagsNumLiteral
} else if name[0] == '"' {
flags |= FlagsLiteral | FlagsStrLiteral
} else {
flags |= FlagsIdent
}
id := ID(key<<KeyShift) | ID(flags)
m.byName[name] = id
m.byKey = append(m.byKey, name)
return id, nil
}
func (m *Map) ByName(name string) ID {
if id, ok := builtInsByName[name]; ok {
return id
}
if m.byName != nil {
return m.byName[name]
}
return 0
}
func (m *Map) ByID(x ID) string { return m.ByKey(Key(x >> KeyShift)) }
func (m *Map) ByToken(t Token) string { return m.ByKey(Key(t.ID >> KeyShift)) }
func (m *Map) ByKey(k Key) string {
if k < nBuiltInKeys {
return builtInsByKey[k].name
}
k -= nBuiltInKeys
if uint(k) < uint(len(m.byKey)) {
return m.byKey[k]
}
return ""
}
func alpha(c byte) bool {
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_')
}
func alphaNumeric(c byte) bool {
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9')
}
func numeric(c byte) bool {
return ('0' <= c && c <= '9')
}
func hasPrefix(a []byte, s string) bool {
if len(s) == 0 {
return true
}
if len(a) < len(s) {
return false
}
a = a[:len(s)]
for i := range a {
if a[i] != s[i] {
return false
}
}
return true
}
func Tokenize(m *Map, filename string, src []byte) (tokens []Token, comments []string, retErr error) {
line := uint32(1)
loop:
for i := 0; i < len(src); {
c := src[i]
if c <= ' ' {
if c == '\n' {
if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() {
tokens = append(tokens, Token{IDSemicolon, line})
}
if line == maxLine {
return nil, nil, fmt.Errorf("token: too many lines in %q", filename)
}
line++
}
i++
continue
}
// TODO: recognize escapes such as `\t`, `\"` and `\\`. For now, we
// assume that strings don't contain control bytes or backslashes.
// Neither should be necessary to parse `use "foo/bar"` lines.
if c == '"' {
j := i + 1
for ; j < len(src); j++ {
c = src[j]
if c == '"' {
j++
break
}
if c == '\\' {
return nil, nil, fmt.Errorf("token: backslash in string at %s:%d", filename, line)
}
if c == '\n' {
return nil, nil, fmt.Errorf("token: expected final '\"' in string at %s:%d", filename, line)
}
if c < ' ' {
return nil, nil, fmt.Errorf("token: control character in string at %s:%d", filename, line)
}
// The -1 is because we still haven't seen the final '"'.
if j-i == maxTokenSize-1 {
return nil, nil, fmt.Errorf("token: string too long at %s:%d", filename, line)
}
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, nil, err
}
tokens = append(tokens, Token{id, line})
i = j
continue
}
if alpha(c) {
j := i + 1
for ; j < len(src) && alphaNumeric(src[j]); j++ {
if j-i == maxTokenSize {
return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line)
}
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, nil, err
}
tokens = append(tokens, Token{id, line})
i = j
continue
}
if numeric(c) {
// TODO: 0x12 hex and 0b11 binary numbers.
//
// TODO: allow underscores like 0b1000_0000_1111?
//
// TODO: reject 0755 octal numbers.
j := i + 1
for ; j < len(src) && numeric(src[j]); j++ {
if j-i == maxTokenSize {
return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line)
}
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, nil, err
}
tokens = append(tokens, Token{id, line})
i = j
continue
}
if c == '/' && i+1 < len(src) && src[i+1] == '/' {
h := i
i += 2
for ; i < len(src) && src[i] != '\n'; i++ {
}
for uint32(len(comments)) < line {
comments = append(comments, "")
}
comments = append(comments, string(src[h:i]))
continue
}
if id := squiggles[c]; id != 0 {
i++
tokens = append(tokens, Token{id, line})
continue
}
for _, x := range lexers[c] {
if hasPrefix(src[i+1:], x.suffix) {
i += len(x.suffix) + 1
tokens = append(tokens, Token{x.id, line})
continue loop
}
}
msg := ""
if c <= 0x7F {
msg = fmt.Sprintf("byte '\\x%02X' (%q)", c, c)
} else {
msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c)
}
return nil, nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line)
}
return tokens, comments, nil
}