From 60a4fdb9fd399210511b265caa02b0a408cd0c74 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 5 May 2017 15:07:30 +1000 Subject: [PATCH] Parse `use "std/io"`. --- lang/ast/ast.go | 7 ++++++- lang/parse/parse.go | 19 +++++++++++++++++ lang/token/list.go | 50 ++++++++++++++++++++++++++------------------- lang/token/token.go | 50 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 95 insertions(+), 31 deletions(-) diff --git a/lang/ast/ast.go b/lang/ast/ast.go index 2e2e587f..96a1587d 100644 --- a/lang/ast/ast.go +++ b/lang/ast/ast.go @@ -112,8 +112,12 @@ const ( // - FlagsSuspendible is "ID1" vs "ID1?" KStruct + // KUse is "use ID1": + // - ID1: package path + KUse + // KFile is a file of source code: - // - List0: top-level declarations + // - List0: top-level declarations KFile ) @@ -139,6 +143,7 @@ var kindStrings = [...]string{ KReturn: "KReturn", KStruct: "KStruct", KType: "KType", + KUse: "KUse", } type Flags uint32 diff --git a/lang/parse/parse.go b/lang/parse/parse.go index 5ea5b5fe..97e39c85 100644 --- a/lang/parse/parse.go +++ b/lang/parse/parse.go @@ -126,6 +126,25 @@ func (p *parser) parseTopLevelDecl() (*a.Node, error) { ID1: id1, List0: list0, }, nil + + case t.IDUse: + p.src = p.src[1:] + id1 := p.peekID() + if !id1.IsStrLiteral() { + got := p.m.ByKey(id1.Key()) + return nil, fmt.Errorf("parse: expected string literal, got %q at %s:%d", got, p.filename, p.line()) + } + p.src = p.src[1:] + if x := p.peekID(); x != t.IDSemicolon { + got := p.m.ByKey(x.Key()) + return nil, fmt.Errorf("parse: expected (implicit) \";\", got %q at %s:%d", got, p.filename, p.line()) + } + p.src = p.src[1:] + return &a.Node{ + Kind: a.KUse, + ID1: id1, + }, nil + } return nil, fmt.Errorf("parse: unrecognized top level declaration at %s:%d", p.filename, p.src[0].Line) } diff --git a/lang/token/list.go b/lang/token/list.go index a93c7445..ae60f936 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -3,7 +3,7 @@ package token -// Flags are the low 12 bits of an ID. For example, they signify whether a +// Flags are the low 14 bits of an ID. For example, they signify whether a // token is an operator, an identifier, etc. // // The flags are not exclusive. For example, the "+" token is both a unary and @@ -14,27 +14,29 @@ package token type Flags uint32 const ( - FlagsOther = Flags(0x001) - FlagsUnaryOp = Flags(0x002) - FlagsBinaryOp = Flags(0x004) - FlagsAssociative = Flags(0x008) - FlagsAssign = Flags(0x010) - FlagsLiteral = Flags(0x020) - FlagsIdent = Flags(0x040) - FlagsImplicitSemicolon = Flags(0x080) - FlagsOpen = Flags(0x100) - FlagsClose = Flags(0x200) - FlagsTightLeft = Flags(0x400) - FlagsTightRight = Flags(0x800) + FlagsOther = Flags(0x0001) + FlagsUnaryOp = Flags(0x0002) + FlagsBinaryOp = Flags(0x0004) + FlagsAssociative = Flags(0x0008) + FlagsLiteral = Flags(0x0010) + FlagsNumLiteral = Flags(0x0020) + FlagsStrLiteral = Flags(0x0040) + FlagsIdent = Flags(0x0080) + FlagsOpen = Flags(0x0100) + FlagsClose = Flags(0x0200) + FlagsTightLeft = Flags(0x0400) + FlagsTightRight = Flags(0x0800) + FlagsAssign = Flags(0x1000) + FlagsImplicitSemicolon = Flags(0x2000) ) -// Key is the high 20 bits of an ID. It is the map key for an IDMap. +// Key is the high 18 bits of an ID. It is the map key for an IDMap. type Key uint32 const ( - idShift = 12 - idMask = 1<<12 - 1 - maxKey = 1<<20 - 1 + idShift = 14 + idMask = 1<<14 - 1 + maxKey = 1<<18 - 1 ) // ID combines a Key and Flags. @@ -46,14 +48,16 @@ func (x ID) Flags() Flags { return Flags(x & idMask) } func (x ID) IsUnaryOp() bool { return Flags(x)&FlagsUnaryOp != 0 } func (x ID) IsBinaryOp() bool { return Flags(x)&FlagsBinaryOp != 0 } func (x ID) IsAssociative() bool { return Flags(x)&FlagsAssociative != 0 } -func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 } func (x ID) IsLiteral() bool { return Flags(x)&FlagsLiteral != 0 } +func (x ID) IsNumLiteral() bool { return Flags(x)&FlagsNumLiteral != 0 } +func (x ID) IsStrLiteral() bool { return Flags(x)&FlagsStrLiteral != 0 } func (x ID) IsIdent() bool { return Flags(x)&FlagsIdent != 0 } -func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 } func (x ID) IsOpen() bool { return Flags(x)&FlagsOpen != 0 } func (x ID) IsClose() bool { return Flags(x)&FlagsClose != 0 } func (x ID) IsTightLeft() bool { return Flags(x)&FlagsTightLeft != 0 } func (x ID) IsTightRight() bool { return Flags(x)&FlagsTightRight != 0 } +func (x ID) IsAssign() bool { return Flags(x)&FlagsAssign != 0 } +func (x ID) IsImplicitSemicolon() bool { return Flags(x)&FlagsImplicitSemicolon != 0 } // Token combines an ID and the line number it was seen. type Token struct { @@ -67,14 +71,16 @@ func (t Token) Flags() Flags { return Flags(t.ID & idMask) } func (t Token) IsUnaryOp() bool { return Flags(t.ID)&FlagsUnaryOp != 0 } func (t Token) IsBinaryOp() bool { return Flags(t.ID)&FlagsBinaryOp != 0 } func (t Token) IsAssociative() bool { return Flags(t.ID)&FlagsAssociative != 0 } -func (t Token) IsAssign() bool { return Flags(t.ID)&FlagsAssign != 0 } func (t Token) IsLiteral() bool { return Flags(t.ID)&FlagsLiteral != 0 } +func (t Token) IsNumLiteral() bool { return Flags(t.ID)&FlagsNumLiteral != 0 } +func (t Token) IsStrLiteral() bool { return Flags(t.ID)&FlagsStrLiteral != 0 } func (t Token) IsIdent() bool { return Flags(t.ID)&FlagsIdent != 0 } -func (t Token) IsImplicitSemicolon() bool { return Flags(t.ID)&FlagsImplicitSemicolon != 0 } func (t Token) IsOpen() bool { return Flags(t.ID)&FlagsOpen != 0 } func (t Token) IsClose() bool { return Flags(t.ID)&FlagsClose != 0 } func (t Token) IsTightLeft() bool { return Flags(t.ID)&FlagsTightLeft != 0 } func (t Token) IsTightRight() bool { return Flags(t.ID)&FlagsTightRight != 0 } +func (t Token) IsAssign() bool { return Flags(t.ID)&FlagsAssign != 0 } +func (t Token) IsImplicitSemicolon() bool { return Flags(t.ID)&FlagsImplicitSemicolon != 0 } // nBuiltInKeys is the number of built-in Keys. The packing is: // - Zero is invalid. @@ -153,6 +159,7 @@ const ( IDBreak = ID(0xA7<> idShift: {"break", IDBreak}, IDContinue >> idShift: {"continue", IDContinue}, IDStruct >> idShift: {"struct", IDStruct}, + IDUse >> idShift: {"use", IDUse}, IDFalse >> idShift: {"false", IDFalse}, IDTrue >> idShift: {"true", IDTrue}, diff --git a/lang/token/token.go b/lang/token/token.go index 2af818d8..e2c644df 100644 --- a/lang/token/token.go +++ b/lang/token/token.go @@ -38,7 +38,9 @@ func (m *IDMap) insert(name string) (ID, error) { } flags := FlagsImplicitSemicolon if numeric(name[0]) { - flags |= FlagsLiteral + flags |= FlagsLiteral | FlagsNumLiteral + } else if name[0] == '"' { + flags |= FlagsLiteral | FlagsStrLiteral } else { flags |= FlagsIdent } @@ -114,11 +116,44 @@ loop: continue } + // TODO: recognize escapes such as `\t`, `\"` and `\\`. For now, we + // assume that strings don't contain control bytes or backslashes. + // Neither should be necessary to parse `use "foo/bar"` lines. + if c == '"' { + j := i + 1 + for ; j < len(src); j++ { + c = src[j] + if c == '"' { + j++ + break + } + if c == '\\' { + return nil, nil, fmt.Errorf("token: backslash in string at %s:%d", filename, line) + } + if c == '\n' { + return nil, nil, fmt.Errorf("token: expected final '\"' in string at %s:%d", filename, line) + } + if c < ' ' { + return nil, nil, fmt.Errorf("token: control character in string at %s:%d", filename, line) + } + // The -1 is because we still haven't seen the final '"'. + if j-i == maxTokenSize-1 { + return nil, nil, fmt.Errorf("token: string too long at %s:%d", filename, line) + } + } + id, err := m.insert(string(src[i:j])) + if err != nil { + return nil, nil, err + } + tokens = append(tokens, Token{id, line}) + i = j + continue + } + if alpha(c) { j := i + 1 - for j < len(src) && alphaNumeric(src[j]) { - j++ - if j-i > maxTokenSize { + for ; j < len(src) && alphaNumeric(src[j]); j++ { + if j-i == maxTokenSize { return nil, nil, fmt.Errorf("token: identifier too long at %s:%d", filename, line) } } @@ -135,12 +170,9 @@ loop: // TODO: 0x12 hex and 0b11 binary numbers. // // TODO: allow underscores like 0b1000_0000_1111? - // - // TODO: save numbers in a separate map, not the IDMap?? j := i + 1 - for j < len(src) && numeric(src[j]) { - j++ - if j-i > maxTokenSize { + for ; j < len(src) && numeric(src[j]); j++ { + if j-i == maxTokenSize { return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line) } }