From 73778b07679dbbbeb71ec750e3d3ba74657240e0 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 5 Nov 2017 16:20:43 +1100 Subject: [PATCH] Re-pack token keys' numeric values. --- lang/token/list.go | 299 ++++++++++++++++++++++----------------------- 1 file changed, 148 insertions(+), 151 deletions(-) diff --git a/lang/token/list.go b/lang/token/list.go index 8447cff3..fe23dafd 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -140,18 +140,18 @@ func (t Token) IsXAssociativeOp() bool { return t.ID.Key().isXOp() && t.ID.IsAss // nBuiltInKeys is the number of built-in Keys. The packing is: // - Zero is invalid. // - [0x01, 0x0F] are reserved for implementation details. -// - [0x10, 0x2F] are squiggly punctuation, such as "(", ")" and ";". -// - [0x30, 0x3F] are squiggly assignments, such as "=" and "+=". -// - [0x40, 0x5F] are operators, such as "+", "==" and "not". -// - [0x60, 0x7F] are keywords, such as "if" and "return". -// - [0x80, 0x8F] are literals, such as "false" and "true". -// - [0x90, 0xCF] are identifiers, such as "bool" and "u32". +// - [0x10, 0x1F] are squiggly punctuation, such as "(", ")" and ";". +// - [0x20, 0x2F] are squiggly assignments, such as "=" and "+=". +// - [0x30, 0x4F] are operators, such as "+", "==" and "not". +// - [0x50, 0x6F] are keywords, such as "if" and "return". +// - [0x70, 0x77] are literals, such as "false" and "true". +// - [0x78, 0xCF] are identifiers, such as "bool", "u32" and "read_u8". // - [0xD0, 0xFF] are disambiguation forms, e.g. unary "+" vs binary "+". // // "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and -// "&=". Their Keys range in [0x01, 0x7F]. -// -// TODO: re-pack the keys. +// "&=". Roughly speaking, their Keys range in [0x01, 0x4F], or disambiguation +// forms range in [0xD0, 0xFF], but vice versa does not necessarily hold. For +// example, the "and" operator is not "squiggly" but it is within [0x01, 0x4F]. const nBuiltInKeys = Key(256) const ( @@ -248,6 +248,11 @@ const ( KeyTrue = Key(IDTrue >> KeyShift) KeyZero = Key(IDZero >> KeyShift) + KeyUnderscore = Key(IDUnderscore >> KeyShift) + KeyThis = Key(IDThis >> KeyShift) + KeyIn = Key(IDIn >> KeyShift) + KeyOut = Key(IDOut >> KeyShift) + KeyI8 = Key(IDI8 >> KeyShift) KeyI16 = Key(IDI16 >> KeyShift) KeyI32 = Key(IDI32 >> KeyShift) @@ -264,14 +269,6 @@ const ( KeyBuf2 = Key(IDBuf2 >> KeyShift) KeyStatus = Key(IDStatus >> KeyShift) - KeyUnderscore = Key(IDUnderscore >> KeyShift) - KeyThis = Key(IDThis >> KeyShift) - KeyIn = Key(IDIn >> KeyShift) - KeyOut = Key(IDOut >> KeyShift) - - KeyLowBits = Key(IDLowBits >> KeyShift) - KeyHighBits = Key(IDHighBits >> KeyShift) - KeyMark = Key(IDMark >> KeyShift) KeyReadU8 = Key(IDReadU8 >> KeyShift) KeyReadU16BE = Key(IDReadU16BE >> KeyShift) @@ -303,6 +300,8 @@ const ( KeyPrefix = Key(IDPrefix >> KeyShift) KeySuffix = Key(IDSuffix >> KeyShift) KeyLimit = Key(IDLimit >> KeyShift) + KeyLowBits = Key(IDLowBits >> KeyShift) + KeyHighBits = Key(IDHighBits >> KeyShift) KeyUnreadU8 = Key(IDUnreadU8 >> KeyShift) KeyXUnaryPlus = Key(IDXUnaryPlus >> KeyShift) @@ -354,144 +353,143 @@ const ( IDOpenCurly = ID(0x14<