Simplify tokenization of multi-byte squiggles.

This commit is contained in:
Nigel Tao committed 2017-05-21 10:56:21 +10:00
1 parent d5d6447ce5
commit 7df75101cd
2 files changed
+8 -26

No files matched your search

-16
View File
@@ -498,22 +498,6 @@ var squiggles = [256]ID{
'?': IDQuestion,
':': IDColon,
';': IDSemicolon,
// 1<<KeyShift is a non-zero ID with zero Flags. It is an invalid ID,
// signifying that lookup should continue in the lexers array below.
'.': 1 << KeyShift,
'!': 1 << KeyShift,
'&': 1 << KeyShift,
'|': 1 << KeyShift,
'^': 1 << KeyShift,
'+': 1 << KeyShift,
'-': 1 << KeyShift,
'*': 1 << KeyShift,
'/': 1 << KeyShift,
'=': 1 << KeyShift,
'<': 1 << KeyShift,
'>': 1 << KeyShift,
}
type suffixLexer struct {
+8 -10
View File
@@ -207,16 +207,14 @@ loop:
if id := squiggles[c]; id != 0 {
i++
if id.Flags() != 0 {
tokens = append(tokens, Token{id, line})
continue
}
for _, x := range lexers[c] {
if hasPrefix(src[i:], x.suffix) {
i += len(x.suffix)
tokens = append(tokens, Token{x.id, line})
continue loop
}
tokens = append(tokens, Token{id, line})
continue
}
for _, x := range lexers[c] {
if hasPrefix(src[i+1:], x.suffix) {
i += len(x.suffix) + 1
tokens = append(tokens, Token{x.id, line})
continue loop
}
}