Tokenize and render comments.

This commit is contained in:
Nigel Tao committed 2017-05-05 12:33:38 +10:00
1 parent d192220c3c
commit 94cb8b25cb
2 files changed
+93 -20

No files matched your search

+75 -8
View File
@@ -18,7 +18,7 @@ import (
var newLine = []byte{'\n'}
func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
func RenderFile(w io.Writer, src []token.Token, comments []string, m *token.IDMap) (err error) {
if len(src) == 0 {
return nil
}
@@ -26,6 +26,7 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
const maxIndent = 0xFFFF
indent := 0
buf := make([]byte, 0, 1024)
commentLine := uint32(0)
prevLine := src[0].Line - 1
prevLineHanging := false
@@ -41,6 +42,25 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
lineTokens := src[:i]
src = src[i:]
// Print any previous comments.
for ; commentLine < line; commentLine++ {
buf = buf[:0]
buf = appendComment(buf, comments, commentLine, indent, true)
if len(buf) == 0 {
continue
}
if commentLine > prevLine+1 {
if _, err = w.Write(newLine); err != nil {
return err
}
}
buf = append(buf, '\n')
if _, err = w.Write(buf); err != nil {
return err
}
prevLine = commentLine
}
// Strip any trailing semi-colons.
hanging := prevLineHanging
prevLineHanging = true
@@ -69,13 +89,7 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
} else if hanging {
indentAdjustment++
}
if n := indent + indentAdjustment; n > 0 {
const tabs = "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"
for ; n > len(tabs); n -= len(tabs) {
buf = append(buf, tabs...)
}
buf = append(buf, tabs[:n]...)
}
buf = appendTabs(buf, indent+indentAdjustment)
// Render the lineTokens.
prevID, prevIsTightRight := token.ID(0), false
@@ -119,13 +133,66 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
prevID = t.ID
}
buf = appendComment(buf, comments, line, 0, false)
buf = append(buf, '\n')
if _, err = w.Write(buf); err != nil {
return err
}
commentLine = line + 1
prevLine = line
prevLineHanging = prevLineHanging && lineTokens[len(lineTokens)-1].ID != token.IDOpenCurly
}
// Print any trailing comments.
for ; uint(commentLine) < uint(len(comments)); commentLine++ {
buf = buf[:0]
buf = appendComment(buf, comments, commentLine, indent, true)
if len(buf) > 0 {
if commentLine > prevLine+1 {
if _, err = w.Write(newLine); err != nil {
return err
}
}
buf = append(buf, '\n')
if _, err = w.Write(buf); err != nil {
return err
}
prevLine = commentLine
}
}
return nil
}
func appendComment(buf []byte, comments []string, line uint32, indent int, otherwiseEmpty bool) []byte {
if uint(line) < uint(len(comments)) {
if com := comments[line]; com != "" {
// Strip trailing spaces.
for len(com) > 0 {
if com[len(com)-1] != ' ' {
break
}
com = com[:len(com)-1]
}
if otherwiseEmpty {
buf = appendTabs(buf, indent)
} else {
buf = append(buf, " "...)
}
buf = append(buf, com...)
}
}
return buf
}
func appendTabs(buf []byte, n int) []byte {
if n > 0 {
const tabs = "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"
for ; n > len(tabs); n -= len(tabs) {
buf = append(buf, tabs...)
}
buf = append(buf, tabs[:n]...)
}
return buf
}
+18 -12
View File
@@ -8,7 +8,10 @@ import (
"fmt"
)
const maxTokenSize = 1023
const (
maxLine = 1048575
maxTokenSize = 1023
)
type IDMap struct {
byName map[string]ID
@@ -91,7 +94,7 @@ func hasPrefix(a []byte, s string) bool {
return true
}
func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, retErr error) {
func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, comments []string, retErr error) {
line := uint32(1)
loop:
for i := 0; i < len(src); {
@@ -102,8 +105,8 @@ loop:
if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() {
tokens = append(tokens, Token{IDSemicolon, line})
}
if line == 1<<32-1 {
return nil, fmt.Errorf("token: too many lines in %q", filename)
if line == maxLine {
return nil, nil, fmt.Errorf("token: too many lines in %q", filename)
}
line++
}
@@ -116,12 +119,12 @@ loop:
for j < len(src) && alphaNumeric(src[j]) {
j++
if j-i > maxTokenSize {
return nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line)
return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line)
}
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, err
return nil, nil, err
}
tokens = append(tokens, Token{id, line})
i = j
@@ -138,12 +141,12 @@ loop:
for j < len(src) && numeric(src[j]) {
j++
if j-i > maxTokenSize {
return nil, fmt.Errorf("token: constant too long at %s:%d", filename, line)
return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line)
}
}
id, err := m.insert(string(src[i:j]))
if err != nil {
return nil, err
return nil, nil, err
}
tokens = append(tokens, Token{id, line})
i = j
@@ -151,11 +154,14 @@ loop:
}
if c == '/' && i+1 < len(src) && src[i+1] == '/' {
// TODO: save comments, dont' just skip them. We'll need to render
// them for puffsfmt.
h := i
i += 2
for ; i < len(src) && src[i] != '\n'; i++ {
}
for uint32(len(comments)) < line {
comments = append(comments, "")
}
comments = append(comments, string(src[h:i]))
continue
}
@@ -180,7 +186,7 @@ loop:
} else {
msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c)
}
return nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line)
return nil, nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line)
}
return tokens, nil
return tokens, comments, nil
}