mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 16:32:11 +01:00
Tokenize and render comments.
This commit is contained in:
1 parent
d192220c3c
commit
94cb8b25cb
2 files changed
+93
-20
No files matched your search
+75
-8
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
var newLine = []byte{'\n'}
|
||||
|
||||
func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
|
||||
func RenderFile(w io.Writer, src []token.Token, comments []string, m *token.IDMap) (err error) {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -26,6 +26,7 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
|
||||
const maxIndent = 0xFFFF
|
||||
indent := 0
|
||||
buf := make([]byte, 0, 1024)
|
||||
commentLine := uint32(0)
|
||||
prevLine := src[0].Line - 1
|
||||
prevLineHanging := false
|
||||
|
||||
@@ -41,6 +42,25 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
|
||||
lineTokens := src[:i]
|
||||
src = src[i:]
|
||||
|
||||
// Print any previous comments.
|
||||
for ; commentLine < line; commentLine++ {
|
||||
buf = buf[:0]
|
||||
buf = appendComment(buf, comments, commentLine, indent, true)
|
||||
if len(buf) == 0 {
|
||||
continue
|
||||
}
|
||||
if commentLine > prevLine+1 {
|
||||
if _, err = w.Write(newLine); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
buf = append(buf, '\n')
|
||||
if _, err = w.Write(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
prevLine = commentLine
|
||||
}
|
||||
|
||||
// Strip any trailing semi-colons.
|
||||
hanging := prevLineHanging
|
||||
prevLineHanging = true
|
||||
@@ -69,13 +89,7 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
|
||||
} else if hanging {
|
||||
indentAdjustment++
|
||||
}
|
||||
if n := indent + indentAdjustment; n > 0 {
|
||||
const tabs = "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"
|
||||
for ; n > len(tabs); n -= len(tabs) {
|
||||
buf = append(buf, tabs...)
|
||||
}
|
||||
buf = append(buf, tabs[:n]...)
|
||||
}
|
||||
buf = appendTabs(buf, indent+indentAdjustment)
|
||||
|
||||
// Render the lineTokens.
|
||||
prevID, prevIsTightRight := token.ID(0), false
|
||||
@@ -119,13 +133,66 @@ func RenderFile(w io.Writer, src []token.Token, m *token.IDMap) (err error) {
|
||||
prevID = t.ID
|
||||
}
|
||||
|
||||
buf = appendComment(buf, comments, line, 0, false)
|
||||
buf = append(buf, '\n')
|
||||
if _, err = w.Write(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
commentLine = line + 1
|
||||
prevLine = line
|
||||
prevLineHanging = prevLineHanging && lineTokens[len(lineTokens)-1].ID != token.IDOpenCurly
|
||||
}
|
||||
|
||||
// Print any trailing comments.
|
||||
for ; uint(commentLine) < uint(len(comments)); commentLine++ {
|
||||
buf = buf[:0]
|
||||
buf = appendComment(buf, comments, commentLine, indent, true)
|
||||
if len(buf) > 0 {
|
||||
if commentLine > prevLine+1 {
|
||||
if _, err = w.Write(newLine); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
buf = append(buf, '\n')
|
||||
if _, err = w.Write(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
prevLine = commentLine
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendComment(buf []byte, comments []string, line uint32, indent int, otherwiseEmpty bool) []byte {
|
||||
if uint(line) < uint(len(comments)) {
|
||||
if com := comments[line]; com != "" {
|
||||
// Strip trailing spaces.
|
||||
for len(com) > 0 {
|
||||
if com[len(com)-1] != ' ' {
|
||||
break
|
||||
}
|
||||
com = com[:len(com)-1]
|
||||
}
|
||||
|
||||
if otherwiseEmpty {
|
||||
buf = appendTabs(buf, indent)
|
||||
} else {
|
||||
buf = append(buf, " "...)
|
||||
}
|
||||
buf = append(buf, com...)
|
||||
}
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func appendTabs(buf []byte, n int) []byte {
|
||||
if n > 0 {
|
||||
const tabs = "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"
|
||||
for ; n > len(tabs); n -= len(tabs) {
|
||||
buf = append(buf, tabs...)
|
||||
}
|
||||
buf = append(buf, tabs[:n]...)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
+18
-12
@@ -8,7 +8,10 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const maxTokenSize = 1023
|
||||
const (
|
||||
maxLine = 1048575
|
||||
maxTokenSize = 1023
|
||||
)
|
||||
|
||||
type IDMap struct {
|
||||
byName map[string]ID
|
||||
@@ -91,7 +94,7 @@ func hasPrefix(a []byte, s string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, retErr error) {
|
||||
func Tokenize(src []byte, m *IDMap, filename string) (tokens []Token, comments []string, retErr error) {
|
||||
line := uint32(1)
|
||||
loop:
|
||||
for i := 0; i < len(src); {
|
||||
@@ -102,8 +105,8 @@ loop:
|
||||
if len(tokens) > 0 && tokens[len(tokens)-1].IsImplicitSemicolon() {
|
||||
tokens = append(tokens, Token{IDSemicolon, line})
|
||||
}
|
||||
if line == 1<<32-1 {
|
||||
return nil, fmt.Errorf("token: too many lines in %q", filename)
|
||||
if line == maxLine {
|
||||
return nil, nil, fmt.Errorf("token: too many lines in %q", filename)
|
||||
}
|
||||
line++
|
||||
}
|
||||
@@ -116,12 +119,12 @@ loop:
|
||||
for j < len(src) && alphaNumeric(src[j]) {
|
||||
j++
|
||||
if j-i > maxTokenSize {
|
||||
return nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line)
|
||||
return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line)
|
||||
}
|
||||
}
|
||||
id, err := m.insert(string(src[i:j]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
tokens = append(tokens, Token{id, line})
|
||||
i = j
|
||||
@@ -138,12 +141,12 @@ loop:
|
||||
for j < len(src) && numeric(src[j]) {
|
||||
j++
|
||||
if j-i > maxTokenSize {
|
||||
return nil, fmt.Errorf("token: constant too long at %s:%d", filename, line)
|
||||
return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line)
|
||||
}
|
||||
}
|
||||
id, err := m.insert(string(src[i:j]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
tokens = append(tokens, Token{id, line})
|
||||
i = j
|
||||
@@ -151,11 +154,14 @@ loop:
|
||||
}
|
||||
|
||||
if c == '/' && i+1 < len(src) && src[i+1] == '/' {
|
||||
// TODO: save comments, dont' just skip them. We'll need to render
|
||||
// them for puffsfmt.
|
||||
h := i
|
||||
i += 2
|
||||
for ; i < len(src) && src[i] != '\n'; i++ {
|
||||
}
|
||||
for uint32(len(comments)) < line {
|
||||
comments = append(comments, "")
|
||||
}
|
||||
comments = append(comments, string(src[h:i]))
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -180,7 +186,7 @@ loop:
|
||||
} else {
|
||||
msg = fmt.Sprintf("non-ASCII byte '\\x%02X'", c)
|
||||
}
|
||||
return nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line)
|
||||
return nil, nil, fmt.Errorf("token: unrecognized %s at %s:%d", msg, filename, line)
|
||||
}
|
||||
return tokens, nil
|
||||
return tokens, comments, nil
|
||||
}
|
||||
Reference in new issue
Block a user