// Copyright 2017 The Wuffs Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package token import ( "errors" "fmt" ) const ( maxID = 1048575 maxLine = 1048575 maxTokenSize = 1023 ) func Unescape(s string) (unescaped string, ok bool) { if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { return "", false } return s[1 : len(s)-1], true } type Map struct { byName map[string]ID byID []string } func (m *Map) Insert(name string) (ID, error) { if name == "" { return 0, nil } if id, ok := builtInsByName[name]; ok { return id, nil } if m.byName == nil { m.byName = map[string]ID{} } if id, ok := m.byName[name]; ok { return id, nil } id := nBuiltInIDs + ID(len(m.byID)) if id > maxID { return 0, errors.New("token: too many distinct tokens") } m.byName[name] = id m.byID = append(m.byID, name) return id, nil } func (m *Map) ByName(name string) ID { if id, ok := builtInsByName[name]; ok { return id } if m.byName != nil { return m.byName[name] } return 0 } func (m *Map) ByID(x ID) string { if x < nBuiltInIDs { return builtInsByID[x] } x -= nBuiltInIDs if uint(x) < uint(len(m.byID)) { return m.byID[x] } return "" } func alpha(c byte) bool { return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') } func alphaNumeric(c byte) bool { return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9') } func hexaNumericUnderscore(c byte) bool { return ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f') || (c == '_') || ('0' <= c && c <= '9') } func numeric(c byte) bool { return ('0' <= c && c <= '9') } func numericUnderscore(c byte) bool { return (c == '_') || ('0' <= c && c <= '9') } func hasPrefix(a []byte, s string) bool { if len(s) == 0 { return true } if len(a) < len(s) { return false } a = a[:len(s)] for i := range a { if a[i] != s[i] { return false } } return true } // checkNumericUnderscores rejects consecutive or trailing underscores. func checkNumericUnderscores(a []byte) bool { prevUnderscore := false for _, c := range a { currUnderscore := c == '_' if prevUnderscore && currUnderscore { return false } prevUnderscore = currUnderscore } return !prevUnderscore } func Tokenize(m *Map, filename string, src []byte) (tokens []Token, comments []string, retErr error) { line := uint32(1) loop: for i := 0; i < len(src); { c := src[i] if c <= ' ' { if c == '\n' { if len(tokens) > 0 && tokens[len(tokens)-1].ID.IsImplicitSemicolon(m) { tokens = append(tokens, Token{IDSemicolon, line}) } if line == maxLine { return nil, nil, fmt.Errorf("token: too many lines in %q", filename) } line++ } i++ continue } // TODO: recognize escapes such as `\t`, `\"` and `\\`. For now, we // assume that strings don't contain control bytes or backslashes. // Neither should be necessary to parse `use "foo/bar"` lines. if c == '"' { j := i + 1 for ; j < len(src); j++ { c = src[j] if c == '"' { j++ break } if c == '\\' { return nil, nil, fmt.Errorf("token: backslash in string at %s:%d", filename, line) } if c == '\n' { return nil, nil, fmt.Errorf("token: expected final '\"' in string at %s:%d", filename, line) } if c < ' ' { return nil, nil, fmt.Errorf("token: control character in string at %s:%d", filename, line) } // The -1 is because we still haven't seen the final '"'. if j-i == maxTokenSize-1 { return nil, nil, fmt.Errorf("token: string too long at %s:%d", filename, line) } } id, err := m.Insert(string(src[i:j])) if err != nil { return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j continue } if alpha(c) { j := i + 1 for ; j < len(src) && alphaNumeric(src[j]); j++ { if j-i == maxTokenSize { return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line) } } id, err := m.Insert(string(src[i:j])) if err != nil { return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j continue } if numeric(c) { // TODO: 0b11 binary numbers. j, isDigit := i+1, numericUnderscore if c == '0' && j < len(src) { if next := src[j]; next == 'x' || next == 'X' { j, isDigit = j+1, hexaNumericUnderscore } else if numeric(next) { return nil, nil, fmt.Errorf("token: legacy octal syntax at %s:%d", filename, line) } } for ; j < len(src) && isDigit(src[j]); j++ { if j-i == maxTokenSize { return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line) } } if !checkNumericUnderscores(src[i:j]) { return nil, nil, fmt.Errorf("token: invalid numeric literal at %s:%d", filename, line) } id, err := m.Insert(string(src[i:j])) if err != nil { return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j continue } if c == '/' && i+1 < len(src) && src[i+1] == '/' { h := i i += 2 for ; i < len(src) && src[i] != '\n'; i++ { } for uint32(len(comments)) < line { comments = append(comments, "") } comments = append(comments, string(src[h:i])) continue } if id := squiggles[c]; id != 0 { i++ tokens = append(tokens, Token{id, line}) continue } for _, x := range lexers[c] { if hasPrefix(src[i+1:], x.suffix) { i += len(x.suffix) + 1 tokens = append(tokens, Token{x.id, line}) continue loop } } msg := "" if c <= 0x7F { msg = fmt.Sprintf("byte '\\x%02X' (%q)", c, c) } else { msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c) } return nil, nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line) } return tokens, comments, nil }