Insert implicit semicolons.

This commit is contained in:
Nigel Tao committed 2017-04-29 11:51:37 +10:00
1 parent a91ae8cdbf
commit 21c6c34d92
2 files changed
+21 -3

No files matched your search

+10 -1
View File
@@ -116,7 +116,16 @@ func init() {
}
}
var operators = [256]ID{
var builtInsImplicitSemicolons = [256]bool{
IDCloseParen: true,
IDCloseBracket: true,
IDCloseCurly: true,
// TODO: break, continue, fallthrough, return, a la Go.
}
// squiggles are built-in tokens that aren't alpha-numeric.
var squiggles = [256]ID{
'(': IDOpenParen,
')': IDCloseParen,
'[': IDOpenBracket,
+11 -2
View File
@@ -8,12 +8,19 @@ import (
"fmt"
)
// nBuiltIns is the number of built-in IDs:
// - The first 128 are squiggles, such as operators and parentheses.
// - The next 128 are keywords.
// - The last 256 aren't returned by Tokenize. The space encodes ambiguous
// 1-byte operators. For example, & might be the start of &^ or &=.
const nBuiltIns = 512
type ID uint32
func (t ID) isBuiltIn() bool { return t < nBuiltIns }
func (t ID) implicitSemicolon() bool { return t >= nBuiltIns || builtInsImplicitSemicolons[t&0xFF] }
type IDMap struct {
byName map[string]ID
byID []string
@@ -125,7 +132,9 @@ loop:
if c <= ' ' {
if c == '\n' {
// TODO: implicit semi-colon insertion a la Go.
if len(tokens) > 0 && tokens[len(tokens)-1].ID.implicitSemicolon() {
tokens = append(tokens, Token{IDSemicolon, line})
}
line++
}
i++
@@ -172,7 +181,7 @@ loop:
continue
}
if id := operators[c]; id != 0 {
if id := squiggles[c]; id != 0 {
i++
if id < lexerBase {
tokens = append(tokens, Token{id, line})