// Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. package token import ( "errors" "fmt" ) const ( maxLine = 1048575 maxTokenSize = 1023 ) type Map struct { byName map[string]ID byKey []string } func (m *Map) insert(name string) (ID, error) { if name == "" { return 0, nil } if id, ok := builtInsByName[name]; ok { return id, nil } if m.byName == nil { m.byName = map[string]ID{} } if id, ok := m.byName[name]; ok { return id, nil } key := nBuiltInKeys + Key(len(m.byKey)) if key > maxKey { return 0, errors.New("token: too many distinct tokens") } flags := FlagsImplicitSemicolon if numeric(name[0]) { flags |= FlagsLiteral | FlagsNumLiteral } else if name[0] == '"' { flags |= FlagsLiteral | FlagsStrLiteral } else { flags |= FlagsIdent } id := ID(key<> KeyShift)) } func (m *Map) ByToken(t Token) string { return m.ByKey(Key(t.ID >> KeyShift)) } func (m *Map) ByKey(k Key) string { if k < nBuiltInKeys { return builtInsByKey[k].name } k -= nBuiltInKeys if uint(k) < uint(len(m.byKey)) { return m.byKey[k] } return "" } func alpha(c byte) bool { return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') } func alphaNumeric(c byte) bool { return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9') } func numeric(c byte) bool { return ('0' <= c && c <= '9') } func hasPrefix(a []byte, s string) bool { if len(s) == 0 { return true } if len(a) < len(s) { return false } a = a[:len(s)] for i := range a { if a[i] != s[i] { return false } } return true } func Tokenize(m *Map, filename string, src []byte) (tokens []Token, comments []string, retErr error) { line := uint32(1) loop: for i := 0; i < len(src); { c := src[i] if c <= ' ' { if c == '\n' { if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() { tokens = append(tokens, Token{IDSemicolon, line}) } if line == maxLine { return nil, nil, fmt.Errorf("token: too many lines in %q", filename) } line++ } i++ continue } // TODO: recognize escapes such as `\t`, `\"` and `\\`. For now, we // assume that strings don't contain control bytes or backslashes. // Neither should be necessary to parse `use "foo/bar"` lines. if c == '"' { j := i + 1 for ; j < len(src); j++ { c = src[j] if c == '"' { j++ break } if c == '\\' { return nil, nil, fmt.Errorf("token: backslash in string at %s:%d", filename, line) } if c == '\n' { return nil, nil, fmt.Errorf("token: expected final '\"' in string at %s:%d", filename, line) } if c < ' ' { return nil, nil, fmt.Errorf("token: control character in string at %s:%d", filename, line) } // The -1 is because we still haven't seen the final '"'. if j-i == maxTokenSize-1 { return nil, nil, fmt.Errorf("token: string too long at %s:%d", filename, line) } } id, err := m.insert(string(src[i:j])) if err != nil { return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j continue } if alpha(c) { j := i + 1 for ; j < len(src) && alphaNumeric(src[j]); j++ { if j-i == maxTokenSize { return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line) } } id, err := m.insert(string(src[i:j])) if err != nil { return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j continue } if numeric(c) { // TODO: 0x12 hex and 0b11 binary numbers. // // TODO: allow underscores like 0b1000_0000_1111? // // TODO: reject 0755 octal numbers. j := i + 1 for ; j < len(src) && numeric(src[j]); j++ { if j-i == maxTokenSize { return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line) } } id, err := m.insert(string(src[i:j])) if err != nil { return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j continue } if c == '/' && i+1 < len(src) && src[i+1] == '/' { h := i i += 2 for ; i < len(src) && src[i] != '\n'; i++ { } for uint32(len(comments)) < line { comments = append(comments, "") } comments = append(comments, string(src[h:i])) continue } if id := squiggles[c]; id != 0 { i++ tokens = append(tokens, Token{id, line}) continue } for _, x := range lexers[c] { if hasPrefix(src[i+1:], x.suffix) { i += len(x.suffix) + 1 tokens = append(tokens, Token{x.id, line}) continue loop } } msg := "" if c <= 0x7F { msg = fmt.Sprintf("byte '\\x%02X' (%q)", c, c) } else { msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c) } return nil, nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line) } return tokens, comments, nil }