From 94cb8b25cb903385c1e502de7fc9f741af0240c0 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 5 May 2017 12:33:38 +1000 Subject: [PATCH] Tokenize and render comments. --- lang/render/render.go | 83 ++++++++++++++++++++++++++++++++++++++----- lang/token/token.go | 30 +++++++++------- 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/lang/render/render.go b/lang/render/render.go index ec1903bc..ad113f1d 100644 --- a/lang/render/render.go +++ b/lang/render/render.go @@ -18,7 +18,7 @@ import ( var newLine = []byte{'\n'} -func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) { +func RenderFile(w io.Writer, src []token.Token, comments []string, m *token.IDMap) (err error) { if len(src) == 0 { return nil } @@ -26,6 +26,7 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) { const maxIndent = 0xFFFF indent := 0 buf := make([]byte, 0, 1024) + commentLine := uint32(0) prevLine := src[0].Line - 1 prevLineHanging := false @@ -41,6 +42,25 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) { lineTokens := src[:i] src = src[i:] + // Print any previous comments. + for ; commentLine < line; commentLine++ { + buf = buf[:0] + buf = appendComment(buf, comments, commentLine, indent, true) + if len(buf) == 0 { + continue + } + if commentLine > prevLine+1 { + if _, err = w.Write(newLine); err != nil { + return err + } + } + buf = append(buf, '\n') + if _, err = w.Write(buf); err != nil { + return err + } + prevLine = commentLine + } + // Strip any trailing semi-colons. hanging := prevLineHanging prevLineHanging = true @@ -69,13 +89,7 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) { } else if hanging { indentAdjustment++ } - if n := indent + indentAdjustment; n > 0 { - const tabs = "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" - for ; n > len(tabs); n -= len(tabs) { - buf = append(buf, tabs...) - } - buf = append(buf, tabs[:n]...) - } + buf = appendTabs(buf, indent+indentAdjustment) // Render the lineTokens. prevID, prevIsTightRight := token.ID(0), false @@ -119,13 +133,66 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) { prevID = t.ID } + buf = appendComment(buf, comments, line, 0, false) buf = append(buf, '\n') if _, err = w.Write(buf); err != nil { return err } + commentLine = line + 1 prevLine = line prevLineHanging = prevLineHanging && lineTokens[len(lineTokens)-1].ID != token.IDOpenCurly } + // Print any trailing comments. + for ; uint(commentLine) < uint(len(comments)); commentLine++ { + buf = buf[:0] + buf = appendComment(buf, comments, commentLine, indent, true) + if len(buf) > 0 { + if commentLine > prevLine+1 { + if _, err = w.Write(newLine); err != nil { + return err + } + } + buf = append(buf, '\n') + if _, err = w.Write(buf); err != nil { + return err + } + prevLine = commentLine + } + } + return nil } + +func appendComment(buf []byte, comments []string, line uint32, indent int, otherwiseEmpty bool) []byte { + if uint(line) < uint(len(comments)) { + if com := comments[line]; com != "" { + // Strip trailing spaces. + for len(com) > 0 { + if com[len(com)-1] != ' ' { + break + } + com = com[:len(com)-1] + } + + if otherwiseEmpty { + buf = appendTabs(buf, indent) + } else { + buf = append(buf, " "...) + } + buf = append(buf, com...) + } + } + return buf +} + +func appendTabs(buf []byte, n int) []byte { + if n > 0 { + const tabs = "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" + for ; n > len(tabs); n -= len(tabs) { + buf = append(buf, tabs...) + } + buf = append(buf, tabs[:n]...) + } + return buf +} diff --git a/lang/token/token.go b/lang/token/token.go index 478e0a38..2af818d8 100644 --- a/lang/token/token.go +++ b/lang/token/token.go @@ -8,7 +8,10 @@ import ( "fmt" ) -const maxTokenSize = 1023 +const ( + maxLine = 1048575 + maxTokenSize = 1023 +) type IDMap struct { byName map[string]ID @@ -91,7 +94,7 @@ func hasPrefix(a []byte, s string) bool { return true } -func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, retErr error) { +func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, comments []string, retErr error) { line := uint32(1) loop: for i := 0; i < len(src); { @@ -102,8 +105,8 @@ loop: if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() { tokens = append(tokens, Token{IDSemicolon, line}) } - if line == 1<<32-1 { - return nil, fmt.Errorf("token: too many lines in %q", filename) + if line == maxLine { + return nil, nil, fmt.Errorf("token: too many lines in %q", filename) } line++ } @@ -116,12 +119,12 @@ loop: for j < len(src) && alphaNumeric(src[j]) { j++ if j-i > maxTokenSize { - return nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line) + return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line) } } id, err := m.insert(string(src[i:j])) if err != nil { - return nil, err + return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j @@ -138,12 +141,12 @@ loop: for j < len(src) && numeric(src[j]) { j++ if j-i > maxTokenSize { - return nil, fmt.Errorf("token: constant too long at %s:%d", filename, line) + return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line) } } id, err := m.insert(string(src[i:j])) if err != nil { - return nil, err + return nil, nil, err } tokens = append(tokens, Token{id, line}) i = j @@ -151,11 +154,14 @@ loop: } if c == '/' && i+1 < len(src) && src[i+1] == '/' { - // TODO: save comments, dont' just skip them. We'll need to render - // them for puffsfmt. + h := i i += 2 for ; i < len(src) && src[i] != '\n'; i++ { } + for uint32(len(comments)) < line { + comments = append(comments, "") + } + comments = append(comments, string(src[h:i])) continue } @@ -180,7 +186,7 @@ loop: } else { msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c) } - return nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line) + return nil, nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line) } - return tokens, nil + return tokens, comments, nil }