From 22be1c0da0f8aa905df6cfc67e167d51c2a2a075 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 8 Feb 2020 15:21:54 +1100 Subject: [PATCH] Allow underscores in numeric constants --- go.mod | 2 +- lang/token/token.go | 30 ++++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index d3f2fc84..bfed505a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/google/wuffs -go 1.12 +go 1.13 require ( github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect diff --git a/lang/token/token.go b/lang/token/token.go index c1e6cedb..b46e4dc3 100644 --- a/lang/token/token.go +++ b/lang/token/token.go @@ -89,14 +89,18 @@ func alphaNumeric(c byte) bool { return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9') } -func hexaNumeric(c byte) bool { - return ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f') || ('0' <= c && c <= '9') +func hexaNumericUnderscore(c byte) bool { + return ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f') || (c == '_') || ('0' <= c && c <= '9') } func numeric(c byte) bool { return ('0' <= c && c <= '9') } +func numericUnderscore(c byte) bool { + return (c == '_') || ('0' <= c && c <= '9') +} + func hasPrefix(a []byte, s string) bool { if len(s) == 0 { return true @@ -113,6 +117,19 @@ func hasPrefix(a []byte, s string) bool { return true } +// checkNumericUnderscores rejects consecutive or trailing underscores. +func checkNumericUnderscores(a []byte) bool { + prevUnderscore := false + for _, c := range a { + currUnderscore := c == '_' + if prevUnderscore && currUnderscore { + return false + } + prevUnderscore = currUnderscore + } + return !prevUnderscore +} + func Tokenize(m *Map, filename string, src []byte) (tokens []Token, comments []string, retErr error) { line := uint32(1) loop: @@ -185,12 +202,10 @@ loop: if numeric(c) { // TODO: 0b11 binary numbers. - // - // TODO: allow underscores like 0b1000_0000_1111? - j, isDigit := i+1, numeric + j, isDigit := i+1, numericUnderscore if c == '0' && j < len(src) { if next := src[j]; next == 'x' || next == 'X' { - j, isDigit = j+1, hexaNumeric + j, isDigit = j+1, hexaNumericUnderscore } else if numeric(next) { return nil, nil, fmt.Errorf("token: legacy octal syntax at %s:%d", filename, line) } @@ -200,6 +215,9 @@ loop: return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line) } } + if !checkNumericUnderscores(src[i:j]) { + return nil, nil, fmt.Errorf("token: invalid numeric literal at %s:%d", filename, line) + } id, err := m.Insert(string(src[i:j])) if err != nil { return nil, nil, err