From 21c6c34d924cc7d3333a60aac113f1190b7451fb Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 29 Apr 2017 11:51:37 +1000 Subject: [PATCH] Insert implicit semicolons. --- lang/token/list.go | 11 ++++++++++- lang/token/token.go | 13 +++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lang/token/list.go b/lang/token/list.go index 974b04f2..0d79e195 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -116,7 +116,16 @@ func init() { } } -var operators = [256]ID{ +var builtInsImplicitSemicolons = [256]bool{ + IDCloseParen: true, + IDCloseBracket: true, + IDCloseCurly: true, + + // TODO: break, continue, fallthrough, return, a la Go. +} + +// squiggles are built-in tokens that aren't alpha-numeric. +var squiggles = [256]ID{ '(': IDOpenParen, ')': IDCloseParen, '[': IDOpenBracket, diff --git a/lang/token/token.go b/lang/token/token.go index c29f26a6..8e43cd40 100644 --- a/lang/token/token.go +++ b/lang/token/token.go @@ -8,12 +8,19 @@ import ( "fmt" ) +// nBuiltIns is the number of built-in IDs: +// - The first 128 are squiggles, such as operators and parentheses. +// - The next 128 are keywords. +// - The last 256 aren't returned by Tokenize. The space encodes ambiguous +// 1-byte operators. For example, & might be the start of &^ or &=. const nBuiltIns = 512 type ID uint32 func (t ID) isBuiltIn() bool { return t < nBuiltIns } +func (t ID) implicitSemicolon() bool { return t >= nBuiltIns || builtInsImplicitSemicolons[t&0xFF] } + type IDMap struct { byName map[string]ID byID []string @@ -125,7 +132,9 @@ loop: if c <= ' ' { if c == '\n' { - // TODO: implicit semi-colon insertion a la Go. + if len(tokens) > 0 && tokens[len(tokens)-1].ID.implicitSemicolon() { + tokens = append(tokens, Token{IDSemicolon, line}) + } line++ } i++ @@ -172,7 +181,7 @@ loop: continue } - if id := operators[c]; id != 0 { + if id := squiggles[c]; id != 0 { i++ if id < lexerBase { tokens = append(tokens, Token{id, line})