From 5c48c84cae1336efbaede09e7f2eda4d338e1351 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 15 Apr 2018 11:41:40 +1000 Subject: [PATCH] Remove token ID/Key distinction, part 6 of N --- lang/token/list.go | 277 +++++++++++++++++++++++---------------------- 1 file changed, 140 insertions(+), 137 deletions(-) diff --git a/lang/token/list.go b/lang/token/list.go index e49694c6..93387406 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -139,12 +139,11 @@ type Token struct { // - [0x10, 0x1F] are squiggly punctuation, such as "(", ")" and ";". // - [0x20, 0x2F] are squiggly assignments, such as "=" and "+=". // - [0x30, 0x4F] are operators, such as "+", "==" and "not". -// - [0x50, 0x6F] are keywords, such as "if" and "return". -// - [0x70, 0x73] are type modifiers, such as "ptr" and "slice". -// - [0x74, 0x77] are literals, such as "false" and "true". -// - [0x78, 0xCF] are identifiers, such as "bool", "u32" and "read_u8". -// - [0xD0, 0xFF] are disambiguation forms, e.g. unary "+" vs binary "+". -// TODO: move the ops and x-ops together. +// - [0x50, 0x7F] are x-ops (disambiguation forms), e.g. unary vs binary "+". +// - [0x80, 0x9F] are keywords, such as "if" and "return". +// - [0xA0, 0xA3] are type modifiers, such as "ptr" and "slice". +// - [0xA4, 0xA7] are literals, such as "false" and "true". +// - [0xA8, 0xFF] are identifiers, such as "bool", "u32" and "read_u8". // // "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and // "&=". Roughly speaking, their Keys range in [0x01, 0x4F], or disambiguation @@ -366,7 +365,9 @@ const ( IDColon = ID(0x1D<